{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    },
    "toc": {
      "nav_menu": {},
      "number_sections": true,
      "sideBar": true,
      "skip_h1_title": false,
      "title_cell": "Table of Contents",
      "title_sidebar": "Contents",
      "toc_cell": false,
      "toc_position": {},
      "toc_section_display": true,
      "toc_window_display": false
    },
    "colab": {
      "name": "gnn-basic-1.ipynb",
      "provenance": []
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "5de85fbdb36e4d4ab4139ec79d1f9ee6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_03db9b100d8d442ea59fdf309f0fd8dd",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_9de989fa2066499999c9896765e06da6",
              "IPY_MODEL_f23fcbf3decd4e92a232636ca058f66c"
            ]
          }
        },
        "03db9b100d8d442ea59fdf309f0fd8dd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9de989fa2066499999c9896765e06da6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5113ba0ed98d4cad9b938cccffb8e7b0",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2c378adbdfd34c0db55e7ff8d8dc4fb1"
          }
        },
        "f23fcbf3decd4e92a232636ca058f66c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_65c323cfadc4470aa8805911e73fbcee",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 9920512/? [00:02&lt;00:00, 4166257.55it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_502021ba036c421bbe8fa7d8648716b0"
          }
        },
        "5113ba0ed98d4cad9b938cccffb8e7b0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2c378adbdfd34c0db55e7ff8d8dc4fb1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "65c323cfadc4470aa8805911e73fbcee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "502021ba036c421bbe8fa7d8648716b0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c4f65f8418064785b61bf68091d462fc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ee92f6f5814c4c8d84989efbff3e262d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_cb5339a352e640ee9f3439a0965f8952",
              "IPY_MODEL_478a968d6b2a495781f5c7f26ecac4f1"
            ]
          }
        },
        "ee92f6f5814c4c8d84989efbff3e262d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cb5339a352e640ee9f3439a0965f8952": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_3576d748087645f7b25e8f779f9b70b4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_568ff4d60f3040fc875473963595ab5a"
          }
        },
        "478a968d6b2a495781f5c7f26ecac4f1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_53ab531e4bca4161a9481c13fe744aa0",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 32768/? [00:01&lt;00:00, 19535.70it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9805845e4f8f4bb1aa6aa61914efa3b9"
          }
        },
        "3576d748087645f7b25e8f779f9b70b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "568ff4d60f3040fc875473963595ab5a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "53ab531e4bca4161a9481c13fe744aa0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9805845e4f8f4bb1aa6aa61914efa3b9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "101e5af3363d46efa9f2abbd2b670d77": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_4e8e1a0047c14a12bcfdbb40fa2ef8d7",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_beb7eac70a8d4a928b5565c2a74289fb",
              "IPY_MODEL_2fe0d3b38c444e2dabf97200622a52cb"
            ]
          }
        },
        "4e8e1a0047c14a12bcfdbb40fa2ef8d7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "beb7eac70a8d4a928b5565c2a74289fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_9fe6e2c30862406db2b6ca08f1d48e64",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_58edac9b2d1c4ae8b9932bdb326b44ab"
          }
        },
        "2fe0d3b38c444e2dabf97200622a52cb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_21bce81c299f4ab5ba2b213715de129d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1654784/? [00:00&lt;00:00, 1657086.92it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_482de32533684ceeb05796006ef7fc1e"
          }
        },
        "9fe6e2c30862406db2b6ca08f1d48e64": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "58edac9b2d1c4ae8b9932bdb326b44ab": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "21bce81c299f4ab5ba2b213715de129d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "482de32533684ceeb05796006ef7fc1e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c446c24a8e6e4a56aecf41011b4ca05e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_2cb09781e1d8492ca144ece72f239328",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_eb3143c7f311440f8c675b62f1a340db",
              "IPY_MODEL_0785c61b9e9845b9a285ee0c3dc77db8"
            ]
          }
        },
        "2cb09781e1d8492ca144ece72f239328": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "eb3143c7f311440f8c675b62f1a340db": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d7fca899bd2149a1b23d3cf936e5d5e7",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2f405188dd454d76a187229d109dc433"
          }
        },
        "0785c61b9e9845b9a285ee0c3dc77db8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_062a9a47c81346a39f7b0a1cbded3589",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 8192/? [00:00&lt;00:00, 11290.95it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5d190926df93477ca3002dceb15a1cdb"
          }
        },
        "d7fca899bd2149a1b23d3cf936e5d5e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2f405188dd454d76a187229d109dc433": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "062a9a47c81346a39f7b0a1cbded3589": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5d190926df93477ca3002dceb15a1cdb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ytlRR9PggzcW",
        "colab_type": "text"
      },
      "source": [
        "Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.\n",
        "- Author: Sebastian Raschka\n",
        "- GitHub Repository: https://github.com/rasbt/deeplearning-models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fBsqxqTWg4IZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install -q IPython\n",
        "!pip install -q ipykernel\n",
        "!pip install -q watermark\n",
        "!pip install -q matplotlib\n",
        "!pip install -q sklearn\n",
        "!pip install -q pandas\n",
        "!pip install -q pydot\n",
        "!pip install -q hiddenlayer\n",
        "!pip install -q graphviz"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hqi8NBepgzcX",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        },
        "outputId": "12740e47-464c-4cb1-9669-8f894bf0a04f"
      },
      "source": [
        "%load_ext watermark\n",
        "%watermark -a 'Sebastian Raschka' -v -p torch"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sebastian Raschka \n",
            "\n",
            "CPython 3.6.9\n",
            "IPython 5.5.0\n",
            "\n",
            "torch 1.5.1+cu101\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tuI2Whw9gzcb",
        "colab_type": "text"
      },
      "source": [
        "- Runs on CPU or GPU (if available)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BVeM3Vzngzcc",
        "colab_type": "text"
      },
      "source": [
        "# Most Basic Graph Neural Network with Gaussian Filter on MNIST"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gNc64y-igzcc",
        "colab_type": "text"
      },
      "source": [
        "Implementing a very basic graph neural network (GNN) using a Gaussian filter. \n",
        "\n",
        "Here, the 28x28 image of a digit in MNIST represents the graph, where each pixel (i.e., cell in the grid) represents a particular node. The feature of that node is simply the pixel intensity in range [0, 1]. \n",
        "\n",
        "Here, the adjacency matrix of the pixels is basically just determined by their neighborhood pixels. Using a Gaussian filter, we connect pixels based on their Euclidean distance in the grid.\n",
        "\n",
        "Using this adjacency matrix $A$, we can compute the output of a layer as \n",
        "\n",
        "$$X^{(l+1)}=A X^{(l)} W^{(l)}.$$\n",
        "\n",
        "Here, $A$ is the $N \\times N$ adjacency matrix, and $X$ is the $N \\times C$ feature matrix (a  2D coordinate array, where $N$ is the total number of pixels -- $28 \\times 28 = 784$ in MNIST). $W$ is the weight matrix of shape $N \\times P$, where $P$ would represent the number of classes if we have only a single hidden layer.\n",
        "\n",
        "\n",
        "\n",
        "- Inspired by and based on Boris Knyazev's tutorial at https://medium.com/@BorisAKnyazev/tutorial-on-graph-neural-networks-for-computer-vision-and-beyond-part-1-3d9fada3b80d."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZCAse0bXgzcd",
        "colab_type": "text"
      },
      "source": [
        "## Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bftLyW_Ogzcd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import time\n",
        "import numpy as np\n",
        "from scipy.spatial.distance import cdist\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torchvision import datasets\n",
        "from torchvision import transforms\n",
        "from torch.utils.data import DataLoader\n",
        "from torch.utils.data.dataset import Subset\n",
        "\n",
        "\n",
        "if torch.cuda.is_available():\n",
        "    torch.backends.cudnn.deterministic = True"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u0-P1wBTgzcg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%matplotlib inline\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bH7QFlT5gzci",
        "colab_type": "text"
      },
      "source": [
        "## Settings and Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L4diEfrNgzci",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "##########################\n",
        "### SETTINGS\n",
        "##########################\n",
        "\n",
        "# Device\n",
        "DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "# Hyperparameters\n",
        "RANDOM_SEED = 1\n",
        "LEARNING_RATE = 0.05\n",
        "NUM_EPOCHS = 20\n",
        "BATCH_SIZE = 128\n",
        "IMG_SIZE = 28\n",
        "\n",
        "# Architecture\n",
        "NUM_CLASSES = 10"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sfurloA6gzcl",
        "colab_type": "text"
      },
      "source": [
        "## MNIST Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QwS08vrUgzcl",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424,
          "referenced_widgets": [
            "5de85fbdb36e4d4ab4139ec79d1f9ee6",
            "03db9b100d8d442ea59fdf309f0fd8dd",
            "9de989fa2066499999c9896765e06da6",
            "f23fcbf3decd4e92a232636ca058f66c",
            "5113ba0ed98d4cad9b938cccffb8e7b0",
            "2c378adbdfd34c0db55e7ff8d8dc4fb1",
            "65c323cfadc4470aa8805911e73fbcee",
            "502021ba036c421bbe8fa7d8648716b0",
            "c4f65f8418064785b61bf68091d462fc",
            "ee92f6f5814c4c8d84989efbff3e262d",
            "cb5339a352e640ee9f3439a0965f8952",
            "478a968d6b2a495781f5c7f26ecac4f1",
            "3576d748087645f7b25e8f779f9b70b4",
            "568ff4d60f3040fc875473963595ab5a",
            "53ab531e4bca4161a9481c13fe744aa0",
            "9805845e4f8f4bb1aa6aa61914efa3b9",
            "101e5af3363d46efa9f2abbd2b670d77",
            "4e8e1a0047c14a12bcfdbb40fa2ef8d7",
            "beb7eac70a8d4a928b5565c2a74289fb",
            "2fe0d3b38c444e2dabf97200622a52cb",
            "9fe6e2c30862406db2b6ca08f1d48e64",
            "58edac9b2d1c4ae8b9932bdb326b44ab",
            "21bce81c299f4ab5ba2b213715de129d",
            "482de32533684ceeb05796006ef7fc1e",
            "c446c24a8e6e4a56aecf41011b4ca05e",
            "2cb09781e1d8492ca144ece72f239328",
            "eb3143c7f311440f8c675b62f1a340db",
            "0785c61b9e9845b9a285ee0c3dc77db8",
            "d7fca899bd2149a1b23d3cf936e5d5e7",
            "2f405188dd454d76a187229d109dc433",
            "062a9a47c81346a39f7b0a1cbded3589",
            "5d190926df93477ca3002dceb15a1cdb"
          ]
        },
        "outputId": "39a9ae4b-3f08-46ab-bda1-cc5b6a39caa9"
      },
      "source": [
        "train_indices = torch.arange(0, 59000)\n",
        "valid_indices = torch.arange(59000, 60000)\n",
        "\n",
        "custom_transform = transforms.Compose([transforms.ToTensor()])\n",
        "\n",
        "\n",
        "train_and_valid = datasets.MNIST(root='data', \n",
        "                                 train=True, \n",
        "                                 transform=custom_transform,\n",
        "                                 download=True)\n",
        "\n",
        "test_dataset = datasets.MNIST(root='data', \n",
        "                              train=False, \n",
        "                              transform=custom_transform,\n",
        "                              download=True)\n",
        "\n",
        "train_dataset = Subset(train_and_valid, train_indices)\n",
        "valid_dataset = Subset(train_and_valid, valid_indices)\n",
        "\n",
        "train_loader = DataLoader(dataset=train_dataset, \n",
        "                          batch_size=BATCH_SIZE,\n",
        "                          num_workers=4,\n",
        "                          shuffle=True)\n",
        "\n",
        "valid_loader = DataLoader(dataset=valid_dataset, \n",
        "                          batch_size=BATCH_SIZE,\n",
        "                          num_workers=4,\n",
        "                          shuffle=False)\n",
        "\n",
        "test_loader = DataLoader(dataset=test_dataset, \n",
        "                         batch_size=BATCH_SIZE,\n",
        "                         num_workers=4,\n",
        "                         shuffle=False)\n",
        "\n",
        "# Checking the dataset\n",
        "for images, labels in train_loader:  \n",
        "    print('Image batch dimensions:', images.shape)\n",
        "    print('Image label dimensions:', labels.shape)\n",
        "    break"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz to data/MNIST/raw/train-images-idx3-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "5de85fbdb36e4d4ab4139ec79d1f9ee6",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting data/MNIST/raw/train-images-idx3-ubyte.gz to data/MNIST/raw\n",
            "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz to data/MNIST/raw/train-labels-idx1-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c4f65f8418064785b61bf68091d462fc",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting data/MNIST/raw/train-labels-idx1-ubyte.gz to data/MNIST/raw\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz to data/MNIST/raw/t10k-images-idx3-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "101e5af3363d46efa9f2abbd2b670d77",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting data/MNIST/raw/t10k-images-idx3-ubyte.gz to data/MNIST/raw\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz to data/MNIST/raw/t10k-labels-idx1-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c446c24a8e6e4a56aecf41011b4ca05e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting data/MNIST/raw/t10k-labels-idx1-ubyte.gz to data/MNIST/raw\n",
            "Processing...\n",
            "Done!\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/pytorch/torch/csrc/utils/tensor_numpy.cpp:141: UserWarning: The given NumPy array is not writeable, and PyTorch does not support non-writeable tensors. This means you can write to the underlying (supposedly non-writeable) NumPy array using the tensor. You may want to copy the array to protect its data or make it writeable before converting it to a tensor. This type of warning will be suppressed for the rest of this program.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Image batch dimensions: torch.Size([128, 1, 28, 28])\n",
            "Image label dimensions: torch.Size([128])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mHMkmyPDgzco",
        "colab_type": "text"
      },
      "source": [
        "## Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ASSfbiRzgzco",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def precompute_adjacency_matrix(img_size):\n",
        "    col, row = np.meshgrid(np.arange(img_size), np.arange(img_size))\n",
        "    \n",
        "    # N = img_size^2\n",
        "    # construct 2D coordinate array (shape N x 2) and normalize\n",
        "    # in range [0, 1]\n",
        "    coord = np.stack((col, row), axis=2).reshape(-1, 2) / img_size\n",
        "\n",
        "    # compute pairwise distance matrix (N x N)\n",
        "    dist = cdist(coord, coord, metric='euclidean')\n",
        "    \n",
        "    # Apply Gaussian filter\n",
        "    sigma = 0.05 * np.pi\n",
        "    A = np.exp(- dist / sigma ** 2)\n",
        "    A[A < 0.01] = 0\n",
        "    A = torch.from_numpy(A).float()\n",
        "\n",
        "    # Normalization as per (Kipf & Welling, ICLR 2017)\n",
        "    D = A.sum(1)  # nodes degree (N,)\n",
        "    D_hat = (D + 1e-5) ** (-0.5)\n",
        "    A_hat = D_hat.view(-1, 1) * A * D_hat.view(1, -1)  # N,N\n",
        "    \n",
        "    return A_hat"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "REtoDFYRgzcs",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 304
        },
        "outputId": "070f1910-dc0f-4fd4-9f38-501c5d3ffb63"
      },
      "source": [
        "plt.imshow(precompute_adjacency_matrix(28));"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQEAAAD8CAYAAAB3lxGOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAYOElEQVR4nO2da4xcZ3nHf8/s+Lr22l4nRBaJmovdRJGInYsgAYRKQqqQIuiHqEqESlRFygfSKggkmqif+qESfCEEqYoaSClIlJuBEkUIGkykClFCEt8CMSG7LihGSczajne9m7U9M08/zJnNeHfOZXbOzLn9f9Jo55wzs3OOx/75fd/zvO/f3B0hRHWpZX0CQohskQSEqDiSgBAVRxIQouJIAkJUHElAiIozFAmY2R1m9rKZTZnZQ8P4DCFEOljadQJmNgb8DrgdOAY8B9zj7i+l+kFCiFQYRkvg3cCUux9193PAt4CPDeFzhBApUB/C73wn8GrX9jHgPctfZGb3A/cD1Oprb9zEBN5orvxtZli9jp8/H/6JZqDKRyEimePUjLtfvHz/MCSQCHd/HHgcYMuai/26Oz/L+C+maJ44ueK1tXXj1C7aROP4DLRWisLqdbAafv7c0M9biKLyU9/7h177h9Ed+CNwWdf2pcG+ULzRZPwXU8y/dydj2ydXHG/Nz9OaO0P9HRdBbazH+xvgLWzN2gFPXYjqMQwJPAfsMrMrzGwtcDfwZNybmidOSgRCZEDqEnD3BvD3wE+AI8B33P03Sd4rEQgxeoZSJ+DuP3L3P3f3q9z9X/p5r0QgxGjJZcWgRCDE6MiHBMxW7OqI4Mz7dzI2MbHieEcEY9sne75/SQT1zG6ACFEIciEBq9epjY+v2N88cZJNz/yWczeGi8Dn5hi7KKJFEPx+IURvciEBP3+e2uZNvUUwO8vaF6Y4d0OICBYX8fn5dotAIhCib3IhAYDG8ZloEeyPEMHCgkQgxCrJjQRoNROJ4Pz1V0kEQqRIfiQAiUSw5sC0WgRCpEi+JABqEQgxYvInAVCLQIgRkk8JwJIIbONGauvXrzi8NFgYdvuwI4JtW0JF4C2XCETlyYcEgjUDVtBq0pyZwTZvprZx44rDnduH56+7MlwEby2Gtgg605IlAlFl8iEBd7Ba73+M7jRPnMTGx0NFsObw0WgRxHQN1CIQVSYfEoD2giBhImg1hyoCWk2JQFSW3EgAJAIhsiBXEgCJQIhRkzsJgEQgxCjJpQQgJRHsTlBQ1GMaskQgqkRuJQAsrR4cKYIN68NFcGg6urJwbk4tAlF5ci0BiKnuazVpnjod3SI4MB3eNVhcxBfeUh2BqDSxEjCzfzez42b26659k2b2tJm9EvzcFuw3M/tSkEF42MxuSOMkY0WQoGvQeFfMGMHkVtURiEqSpCXwH8Ady/Y9BOxz913AvmAb4MPAruBxP/BYOqc5uAjqL7ZFUNu8eeXbFxZiWwQSgSgrsRJw9/8BlscCfQz4WvD8a8Bfd+3/urf5JbDVzHakdbKDimDs8DSNJIOFEoGoEKsdE7jE3V8Lnr8OXBI875VD+M5ev8DM7jez583s+fOcTfzBg4igNTc3UNdAIhBlZOCBQW9nm/edBuruj7v7Te5+0xrW9ffeFLoG569T10AIWL0E3ug084Ofx4P9fecQrpaOCML+oS6JIGQacv3QNI09O8NFoDoCURFWK4EngXuD5/cCP+za/4ngLsHNwOmubkPqeKOB1WKmIW8K7xrUD07RiFiYpDU7G7qcuUQgykKSW4TfBP4XuNrMjpnZfcDngNvN7BXgQ8E2wI+Ao8AU8GXgk0M56y4iWwTuNE++GT1GEFFH4GfParBQlJ7Yv73ufk/Iodt6vNaBBwY9qX7xRgOr13HGlgp8lgi6BmOTW6nR/h++m+46gvqLR2nOzl749oUFasDY9kmaJ072/P3OWPvzO0ISokDkvmIwKZ2uQegYQUSLoHuwULcPRdUojQQggQiGWFkoEYiiUioJQEIRbNwQW1kYKgLdPhQlo3QSAHUNhOiHUkoAkncNbN3KQqWl9Qiuvyq6jmBya2gdAWj2oSgGpZUAJKsjqE1MhLcI9kfXEfiZ+chYdLUIRBEotQQgQR1B3FyDA9PhYwSdWHQNFooCU3oJwOB3DTQNWZSZSkgABheBpiGLslIZCcBgtw81DVmUlUpJANK7faiugSgLlZMApDBGcGia5nXqGohyUEkJQB9dgx7rEbTm5totgrg6gm1bItcj6PnZQoyYykoAEoogIha9vn+Kxp6IOoL5hcj1CEAFRSJ7Ki0BiBFBp44garDwUII6AnUNRI6pvARAcw1EtZEEAjQNWVQVSaALTUMWVUQSWEYaXYPYFoFEIHJEkoVGLzOzZ8zsJTP7jZk9GOwfaR7hKImdfZgw10Cx6KIIJGkJNIDPuPu1wM3AA2Z2LRnkEY6SRLkGG9aH5hp01iOIjEWf3KYWgcicJFmEr7n7/uD5HHCEdrRYJnmEoyS+a3Aqso4gNhb9rUXFoovM6WtMwMwuB64HnmXAPMLVZhGOmkR1BAnuGsQmHWlhEpERiSVgZpuA7wGfcvcLFudfTR7hIFmEoyaNacixcw10+1BkRCIJmNka2gL4hrt/P9ideR7hKBl0GrJuH4q8kuTugAFPAEfc/Qtdh3KRRzhKVFkoykiSlsD7gL8FbjWzg8HjTnKURzhKVFkoykaSLMKfAz1uaAM5yiMcJYmyD7dPUmu1aC0uXnB4qUWw+yrqB6dozc1d+Pbu7MOZGfBlQy3KPhQpo4rBVRJbRzAzg41vDO8axMSi+9ycugZiJEgCg9BqRt8+PHV6sOXMNVgoRoAkMCBpLGeuwUKRJZJACgw6+zDRYKFEIIaEJJASmn0oiookkCJpJR3p9qEYJZJAymhhElE0JIEhkLhrEDINOdFy5hHrEYBmH4rkSAJDIkksum2K6BrExKK3ZmcViy5SQRIYIrGx6BGDhXF1BH72rAYLRSpIAkNGg4Ui70gCIyAtESgEVQwDSWBEpLEwSWO3QlBF+kgCI2TQhUk0DVkMA0lgxAx9YRJ1DUSfSAIZkLRrYOtWrr3YvZx5ZB3B5FbVEYhESAIZESuCmRlqExOrj0U/M686ApEISSBDBlnOvBOLHplroMFCkYAkC42uN7NfmdmhIIbsn4P9V5jZs0Hc2LfNbG2wf12wPRUcv3y4l1BsNA1ZZE2SlsBZ4FZ33w3sAe4IVhH+PPCIu+8ETgH3Ba+/DzgV7H8keJ2IQKsYiyxJEkPm7n4m2FwTPBy4Fdgb7F8eQ9aJJ9sL3BYsWy4i0CrGIiuSho+MmdlB2gEjTwPTwJvu3lnutjtqbCmGLDh+Gtie5kmXFU1DFlmQSALu3nT3PbTThN4NXDPoBxcli3DUxM4+VNdApExfdwfc/U3gGeAW2mnDnb8t3VFjSzFkwfEtwIkev6swWYSjJlEs+sYNq49Fn59nbNuWSBH0PCZKSZK7Axeb2dbg+Qbgdtrx5M8AdwUvWx5D1oknuwv4WRBIIvogUddgtbHoCwuxseihrRFROpK0BHYAz5jZYeA54Gl3fwr4R+DTZjZFu8//RPD6J4Dtwf5PAw+lf9rVIK1Y9NUMFqqgqDpYHv6TnrBJf4+tSDQTAVavt5voyyPPAGpjjG2fxOfnaS0srDg8NjHRFsHh6RWRZwC1jRux8XGaJ06G/n6rmSLPSsBPfe8L7n7T8v2qGCwAmoYshokkUBA0DVkMC0mgQKRVWagVikQ3kkDBiK0jiFuq7FDCrkHINGSJoHxIAgUkUR3BhvWRXYPIOgLFolcKSaCgxHYNImLRY+sIEsSigxYmKQuSQIHJchqy6gjKgyRQcDQNWQyKJFACNA1ZDIIkUBI0DVmsFkmgRKhrIFaDJFAyEncNoqYh7159LLpEUDwkgRLSqSNYdSz6wdXHoksExUMSKCudGYHDjEXXYGEpkARKjNKQRRIkgZKTxjTk5nWahlxmJIEKMOg05Njbh+oaFBpJoCIoDVmEIQlUiJFUFkoEhSOxBIIAkgNm9lSwrSzCAjJoLHr9RcWil41+WgIP0l5qvIOyCAtK7MIkSWLRo+oI5s4oFr1AJI0huxT4K+ArwbahLMJCE7kwSWc586g1C5PUEahrUAiStgS+CHwWaAXb2xkwi1AxZNmjuQYCkiUQfQQ47u4vpPnBiiHLB5qGLJK0BN4HfNTMfg98i3Y34FEGzCIU+UHTkKtNrATc/WF3v9TdLwfupp0t+HGURVgq0ugaqEVQTAapE1AWYclIa66BWgTFQlmEYgWx2YeTW2nNncHPrhzQHZuY4Pz1V1HfPxWefbhhPc2Tp6DH373IzxYDoSxCkZgkLYLYOoI94XUEPr8QWUcAKigaJZKA6MkgseituTnWHJoOn4a8uKjbhzlCEhChjGQasgYLM0cSEJEMfRqyBgszRxIQsaiysNxIAiIRI4k8U9cgEyQBkZjY2YdamKSQSAKiLxLHokflGsTFom/bIhGMEElA9E38GMEpbPPm1ceiv7WoWPQRIgmIVTFIHcFSi2CVg4VamCRdJAGxakYyWKgxgqEjCYiB0O3D4iMJiIHRwiTFRhIQqaCFSYqLJCBSI3EdQcjtw8RdA8Wip4okIFKlM3If3TXYGH3XIK6OQC2CVJEERPq0mtFdg1Ono8cI4uoIYroGoDqCfpAExFDIcrBQdQT9IQmIoZHWmoWhkWcaLEyFpAlEvzezF83soJk9H+ybNLOnzeyV4Oe2YL+Z2ZeCLMLDZnbDMC9A5Js0FiZp7I5ZmEQiGIh+WgIfdPc9XQsVPgTsc/ddwD7eXlX4w8Cu4HE/8FhaJyuKyaALk6iOYLgM0h3ozhxcnkX4dW/zS9ohJTsG+BxRAtKqLFTXIH2SSsCB/zazF8zs/mDfJe7+WvD8deCS4PlSFmFAd07hEsoirB6JuwZhdQSHpmns2Rkdi646gr5JKoH3u/sNtJv6D5jZB7oPBglDfQUYKIuwmiSJRbdN4asY1w/GxKLPzoYuZy4R9CaRBNz9j8HP48APgHcDb3Sa+cHP48HLl7IIA7pzCoWIj0WP6Bp0YtHD6ggUi94/SVKJx81sc+c58JfAr7kwc3B5FuEngrsENwOnu7oNQgCahpwnkvwpXAL8wNr9rDrwn+7+YzN7DviOmd0H/AH4m+D1PwLuBKaABeDvUj9rUQq80WjHjjG2MnYsGCwc2z5JjfY/7G66BwvXHD5Kc3b2wrcvLFADxrZP0jxxsufvd8ban99pmVQUZRGKzInNPtw+ic/PrxABtLMPG++6kvqLK0UAQfbhxg00T74Z+vutZpUQgbIIRW7RNORskQRELtAKRdkhCYjckLSOwNatvKXcPQ05so5gcmtoHQFUc/ahJCByRZI6gthY9Ig6Aj8zHxmLXsUWgSQgckdsHUFcLPqB6fAxgk4suuYaLCEJiFyiacijQxIQuUXTkEeDJCByjaYhDx9JQOQeTUMeLpKAKAQDjxEcmqZ5nboGvZAERGFI3DXosR5Ba26u3SKIqyPYtiVyPYKen11wJAFRKBKJICIWvb5/isaeiDqC+YXI9QigfAVFkoAoHIli0aMGCw8lqCOoUNdAEhCFRHMN0kMSEIVFacjpIAmIQqNpyIMjCYjCk0bXoMpLlUkCohTEzj5MMNegqrHokoAoDZGzDzsi2LA+NNcgUSz65LbStQiSZhFuNbO9ZvZbMztiZrcoi1DkkfiuwanIOoLYWPS3FksXi560JfAo8GN3vwbYDRxBWYQipySqI0hw1yA26agkC5MkyR3YAnwAeALA3c+5+5soi1DkmDSmIcfONSjJ7cMkLYErgD8BXzWzA2b2lSCERFmEItcMOg25KrcPk0igDtwAPObu1wPzvN30B5RFKPKLKgvjSSKBY8Axd3822N5LWwrKIhSFQJWF0cRKwN1fB141s6uDXbcBL6EsQlEgBo5Ff/Eo53dHTEPudA0KWEeQ9Kz+AfiGma0FjtLOF6yhLEJRILzRaEugFpJ9ODPD2OQ2arVa7+zDIBZ9zYHp3tmHrVY7+7BX5FmOsw8TScDdDwIrMsxotwqWv9aBBwY8LyGGQ6sZHoLqTvPU6dAQ1O7lzHtlH7YWF6nVaoULQVXFoKgcaSxnXqbBQklAVJJBZx8mGiwsiAgkAVFZNPuwjSQgKk1aSUdFvn0oCYjKU/WFSSQBIeijaxBVRxC3nHlEHQFkN/tQEhAiIEksum2K6BrExKK3ZmdzGYsuCQjRRWwsesRgYVwsup89m8vBQklAiGVUbbBQEhCiB2mJoAghqJKAECGksTBJY3f+Q1AlASEiGHRhkiJMQ5YEhIhh6AuTZNw1kASESEDSroGtW7lKVvdy5pF1BJNbM6kjkASESEisCGZmqE1MrD4W/cx8JnUEkoAQfTDIcuadWPTIXIOYwUJIv0UgCQjRJ1lOQ+4UM6UpAklAiFWQ5SrGaYsgSfjI1WZ2sOsxa2afUgyZqDojWcV4BCJIstrwy+6+x933ADfSXjz0ByiGTIjhT0MegQj67Q7cBky7+x9QDJkQQILZhznvGvQrgbuBbwbPB4ohE6JMJIpF37hh9bHo8/OMbdsylNuHiSUQZA58FPjuipNYRQyZsghF2UjUNYiLRY+aa5AkFn3N2r7Pu5+WwIeB/e7+RrA9UAyZsghFGRk4Fj2qjiBJ18BbfYugHwncw9tdAVAMmRA9SeOuwShFkEgCQRT57cD3u3Z/DrjdzF4BPhRsQzuG7CjtGLIvA59MfDZClIQiiSBpDNk8sH3ZvhMohkyIULzRCI88C0QQFnnWLYI1h3tEni0sUIPQyLPOZ9uatfj5c5HnqYpBIYZIEVoEkoAQQya2jiAtEfSYhpxEBJKAECMgUR3BhvWRIjh3407Gtm5Z+faFBXxujrF3XBxeR9BsrtjfQRIQYkTEtghOnY5sEaw9MM25PSF1BIuL+Jl56u/ovR7BijGJLiQBIUZIZJlvXNfgzdOs3T8VXlk4P09r7ky4CEKQBIQYMQOJIKgsPBeWdLQKEUgCQmTAoCJIs0UgCQiREXlpEUgCQmRIGi2CQUUgCQiRMYluH0bEoq/dP9W+fRglgh2XrDjWQRIQIgckikXfMkFtfHzF4ebsLGtfmOLMB69pFw0tf/v8PM2ZE6GfLQkIkRO80QCr9RaBO80/naC2eVOoCDb9fIr59+7sKQI/G75mhyQgRI7w8+fCRdBq0jg+Ey6CEycZ/0W4CMKQBITIGaMWgSQgRA4ZpQgkASFyyqhEIAkIkWNGIQJJQIic0xFBWB1B4/gMta1beseiByI4cefVob/f2quBZYuZzQEvZ30eQ+QiYCbrkxgSZb42KNf1/Zm7X7x8Z/ph56vjZXe/KeuTGBZm9nxZr6/M1wblvz5Qd0CIyiMJCFFx8iKBx7M+gSFT5usr87VB+a8vHwODQojsyEtLQAiREZKAEBUncwmY2R1m9rKZTZnZQ1mfT7+Y2WVm9oyZvWRmvzGzB4P9k2b2tJm9EvzcFuw3M/tScL2HzeyGbK8gHjMbM7MDZvZUsH2FmT0bXMO3g9h6zGxdsD0VHL88y/NOgpltNbO9ZvZbMztiZreU6btLQqYSMLMx4F9px55fC9xjZtdmeU6roAF8xt2vBW4GHgiu4SFgn7vvAvYF29C+1l3B437gsdGfct88CBzp2v488Ii77wROAfcF++8DTgX7Hwlel3ceBX7s7tcAu2lfZ5m+u3jcPbMHcAvwk67th4GHszynFK7ph7QTnF8GdgT7dtAuiAL4N+CertcvvS6PD+BS2v8QbgWeAox2BV19+XcI/AS4JXheD15nWV9DxLVtAf5v+TmW5btL+si6O/BO4NWu7WPBvkISNH+vB54FLnH314JDrwOdRd6Kds1fBD4LtILt7cCb7h4sjHfB+S9dW3D8NMvSrHPGFcCfgK8G3Z2vmNk45fnuEpG1BEqDmW0Cvgd8yt0vyJH29n8bhbsXa2YfAY67+wtZn8uQqAM3AI+5+/XAPG83/YHifnf9kLUE/ghc1rV9abCvUJjZGtoC+Ia7fz/Y/YaZ7QiO7wCOB/uLdM3vAz5qZr8HvkW7S/AosNXMOvNOus9/6dqC41uA8BUus+cYcMzdnw2299KWQhm+u8RkLYHngF3BaPNa4G7gyYzPqS/MzIAngCPu/oWuQ08C9wbP76U9VtDZ/4lgpPlm4HRX0zNXuPvD7n6pu19O+7v5mbt/HHgGuCt42fJr61zzXcHrc/u/qLu/DrxqZp15trcBL1GC764vsh6UAO4EfgdMA/+U9fms4vzfT7u5eBg4GDzupN0X3ge8AvwUmAxeb7TviEwDLwI3ZX0NCa/zL4CngudXAr8CpoDvAuuC/euD7ang+JVZn3eC69oDPB98f/8FbCvbdxf3UNmwEBUn6+6AECJjJAEhKo4kIETFkQSEqDiSgBAVRxIQouJIAkJUnP8H3SPOLb2N4bQAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bkznEufcgzcu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "##########################\n",
        "### MODEL\n",
        "##########################\n",
        "\n",
        "        \n",
        "\n",
        "class GraphNet(nn.Module):\n",
        "    def __init__(self, img_size=28, num_classes=10):\n",
        "        super(GraphNet, self).__init__()\n",
        "        \n",
        "        n_rows = img_size**2\n",
        "        self.fc = nn.Linear(n_rows, num_classes, bias=False)\n",
        "\n",
        "        A = precompute_adjacency_matrix(img_size)\n",
        "        self.register_buffer('A', A)\n",
        "\n",
        "        \n",
        "\n",
        "    def forward(self, x):\n",
        "        \n",
        "        B = x.size(0) # Batch size\n",
        "\n",
        "        ### Reshape Adjacency Matrix\n",
        "        # [N, N] Adj. matrix -> [1, N, N] Adj tensor where N = HxW\n",
        "        A_tensor = self.A.unsqueeze(0)\n",
        "        # [1, N, N] Adj tensor -> [B, N, N] tensor\n",
        "        A_tensor = self.A.expand(B, -1, -1)\n",
        "        \n",
        "        ### Reshape inputs\n",
        "        # [B, C, H, W] => [B, H*W, 1]\n",
        "        x_reshape = x.view(B, -1, 1)\n",
        "        \n",
        "        # bmm = batch matrix product to sum the neighbor features\n",
        "        # Input: [B, N, N] x [B, N, 1]\n",
        "        # Output: [B, N]\n",
        "        avg_neighbor_features = (torch.bmm(A_tensor, x_reshape).view(B, -1))\n",
        "        \n",
        "        logits = self.fc(avg_neighbor_features)\n",
        "        probas = F.softmax(logits, dim=1)\n",
        "        return logits, probas"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rRFbFWlkgzcw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "torch.manual_seed(RANDOM_SEED)\n",
        "model = GraphNet(img_size=IMG_SIZE, num_classes=NUM_CLASSES)\n",
        "\n",
        "model = model.to(DEVICE)\n",
        "\n",
        "optimizer = torch.optim.SGD(model.parameters(), lr=LEARNING_RATE)  "
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nTRExg4UhARX",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 689
        },
        "outputId": "5d1f01a5-8c29-42ae-8f7b-1d93eb536a70"
      },
      "source": [
        "import hiddenlayer as hl\n",
        "hl.build_graph(model, torch.zeros([128, 1, 28, 28]).to(DEVICE))"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<hiddenlayer.graph.Graph at 0x7f03ca43ff60>"
            ],
            "image/svg+xml": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n -->\n<!-- Title: %3 Pages: 1 -->\n<svg width=\"1788pt\" height=\"486pt\"\n viewBox=\"0.00 0.00 1788.00 486.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(72 450)\">\n<title>%3</title>\n<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-72,36 -72,-450 1716,-450 1716,36 -72,36\"/>\n<!-- /outputs/3 -->\n<g id=\"node1\" class=\"node\">\n<title>/outputs/3</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"54,-279 0,-279 0,-243 54,-243 54,-279\"/>\n<text text-anchor=\"start\" x=\"15\" y=\"-258\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Shape</text>\n</g>\n<!-- /outputs/5 -->\n<g id=\"node3\" class=\"node\">\n<title>/outputs/5</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"145,-252 91,-252 91,-216 145,-216 145,-252\"/>\n<text text-anchor=\"start\" x=\"104\" y=\"-231\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Gather</text>\n</g>\n<!-- /outputs/3&#45;&gt;/outputs/5 -->\n<g id=\"edge1\" class=\"edge\">\n<title>/outputs/3&#45;&gt;/outputs/5</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M54.303,-252.8991C62.6713,-250.4162 72.0227,-247.6416 80.9279,-244.9994\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"82.1249,-248.2951 90.7163,-242.0952 80.1338,-241.5843 82.1249,-248.2951\"/>\n</g>\n<!-- /outputs/4 -->\n<g id=\"node2\" class=\"node\">\n<title>/outputs/4</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"54,-225 0,-225 0,-189 54,-189 54,-225\"/>\n<text text-anchor=\"start\" x=\"9\" y=\"-204\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Constant</text>\n</g>\n<!-- /outputs/4&#45;&gt;/outputs/5 -->\n<g id=\"edge2\" class=\"edge\">\n<title>/outputs/4&#45;&gt;/outputs/5</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M54.303,-215.1009C62.6713,-217.5838 72.0227,-220.3584 80.9279,-223.0006\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"80.1338,-226.4157 90.7163,-225.9048 82.1249,-219.7049 80.1338,-226.4157\"/>\n</g>\n<!-- /outputs/8 -->\n<g id=\"node6\" class=\"node\">\n<title>/outputs/8</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"243,-198 182,-198 182,-162 243,-162 243,-198\"/>\n<text text-anchor=\"start\" x=\"190.5\" y=\"-177\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Unsqueeze</text>\n</g>\n<!-- /outputs/5&#45;&gt;/outputs/8 -->\n<g id=\"edge3\" class=\"edge\">\n<title>/outputs/5&#45;&gt;/outputs/8</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M145.0799,-218.5258C153.7595,-213.566 163.5569,-207.9675 172.9387,-202.6064\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"174.8104,-205.5681 181.7563,-197.5678 171.3374,-199.4904 174.8104,-205.5681\"/>\n</g>\n<!-- /outputs/23 -->\n<g id=\"node21\" class=\"node\">\n<title>/outputs/23</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"243,-252 182,-252 182,-216 243,-216 243,-252\"/>\n<text text-anchor=\"start\" x=\"190.5\" y=\"-231\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Unsqueeze</text>\n</g>\n<!-- /outputs/5&#45;&gt;/outputs/23 -->\n<g id=\"edge4\" class=\"edge\">\n<title>/outputs/5&#45;&gt;/outputs/23</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M145.0799,-234C153.3121,-234 162.5498,-234 171.4848,-234\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"171.7564,-237.5001 181.7563,-234 171.7563,-230.5001 171.7564,-237.5001\"/>\n</g>\n<!-- /outputs/30 -->\n<g id=\"node28\" class=\"node\">\n<title>/outputs/30</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"243,-306 182,-306 182,-270 243,-270 243,-306\"/>\n<text text-anchor=\"start\" x=\"190.5\" y=\"-285\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Unsqueeze</text>\n</g>\n<!-- /outputs/5&#45;&gt;/outputs/30 -->\n<g id=\"edge5\" class=\"edge\">\n<title>/outputs/5&#45;&gt;/outputs/30</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M145.0799,-249.4742C153.7595,-254.434 163.5569,-260.0325 172.9387,-265.3936\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"171.3374,-268.5096 181.7563,-270.4322 174.8104,-262.4319 171.3374,-268.5096\"/>\n</g>\n<!-- /outputs/6 -->\n<g id=\"node4\" class=\"node\">\n<title>/outputs/6</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"145,-36 91,-36 91,0 145,0 145,-36\"/>\n<text text-anchor=\"start\" x=\"100\" y=\"-15\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Constant</text>\n</g>\n<!-- /outputs/9 -->\n<g id=\"node7\" class=\"node\">\n<title>/outputs/9</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"243,-36 182,-36 182,0 243,0 243,-36\"/>\n<text text-anchor=\"start\" x=\"190.5\" y=\"-15\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Unsqueeze</text>\n</g>\n<!-- /outputs/6&#45;&gt;/outputs/9 -->\n<g id=\"edge6\" class=\"edge\">\n<title>/outputs/6&#45;&gt;/outputs/9</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M145.0799,-18C153.3121,-18 162.5498,-18 171.4848,-18\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"171.7564,-21.5001 181.7563,-18 171.7563,-14.5001 171.7564,-21.5001\"/>\n</g>\n<!-- /outputs/7 -->\n<g id=\"node5\" class=\"node\">\n<title>/outputs/7</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"145,-90 91,-90 91,-54 145,-54 145,-90\"/>\n<text text-anchor=\"start\" x=\"100\" y=\"-69\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Constant</text>\n</g>\n<!-- /outputs/10 -->\n<g id=\"node8\" class=\"node\">\n<title>/outputs/10</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"243,-90 182,-90 182,-54 243,-54 243,-90\"/>\n<text text-anchor=\"start\" x=\"190.5\" y=\"-69\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Unsqueeze</text>\n</g>\n<!-- /outputs/7&#45;&gt;/outputs/10 -->\n<g id=\"edge7\" class=\"edge\">\n<title>/outputs/7&#45;&gt;/outputs/10</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M145.0799,-72C153.3121,-72 162.5498,-72 171.4848,-72\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"171.7564,-75.5001 181.7563,-72 171.7563,-68.5001 171.7564,-75.5001\"/>\n</g>\n<!-- /outputs/11 -->\n<g id=\"node9\" class=\"node\">\n<title>/outputs/11</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"334,-91 280,-91 280,-55 334,-55 334,-91\"/>\n<text text-anchor=\"start\" x=\"292\" y=\"-70\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Concat</text>\n</g>\n<!-- /outputs/8&#45;&gt;/outputs/11 -->\n<g id=\"edge8\" class=\"edge\">\n<title>/outputs/8&#45;&gt;/outputs/11</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M234.4361,-161.7248C237.4241,-158.912 240.3655,-155.9568 243,-153 262.1111,-131.5512 261.5496,-122.0197 280,-100 280.3858,-99.5395 280.7782,-99.0778 281.1761,-98.6155\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"283.8051,-100.9269 287.9614,-91.1813 278.6349,-96.208 283.8051,-100.9269\"/>\n</g>\n<!-- /outputs/9&#45;&gt;/outputs/11 -->\n<g id=\"edge9\" class=\"edge\">\n<title>/outputs/9&#45;&gt;/outputs/11</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M243.1814,-35.8569C252.0738,-41.0324 261.8627,-46.7296 271.0467,-52.0748\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"269.5701,-55.265 279.9735,-57.2703 273.0913,-49.2151 269.5701,-55.265\"/>\n</g>\n<!-- /outputs/10&#45;&gt;/outputs/11 -->\n<g id=\"edge10\" class=\"edge\">\n<title>/outputs/10&#45;&gt;/outputs/11</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M243.1814,-72.3247C251.7033,-72.4149 261.0485,-72.5137 269.8957,-72.6074\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"269.937,-76.1079 279.9735,-72.714 270.0111,-69.1083 269.937,-76.1079\"/>\n</g>\n<!-- /outputs/13 -->\n<g id=\"node11\" class=\"node\">\n<title>/outputs/13</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"425,-145 371,-145 371,-109 425,-109 425,-145\"/>\n<text text-anchor=\"start\" x=\"381\" y=\"-124\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Reshape</text>\n</g>\n<!-- /outputs/11&#45;&gt;/outputs/13 -->\n<g id=\"edge11\" class=\"edge\">\n<title>/outputs/11&#45;&gt;/outputs/13</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M334.303,-89.2018C343.0312,-94.3811 352.829,-100.1952 362.0744,-105.6815\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"360.3303,-108.7164 370.7163,-110.8097 363.9025,-102.6965 360.3303,-108.7164\"/>\n</g>\n<!-- /outputs/12 -->\n<g id=\"node10\" class=\"node\">\n<title>/outputs/12</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"334,-145 280,-145 280,-109 334,-109 334,-145\"/>\n<text text-anchor=\"start\" x=\"289\" y=\"-124\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Constant</text>\n</g>\n<!-- /outputs/12&#45;&gt;/outputs/13 -->\n<g id=\"edge12\" class=\"edge\">\n<title>/outputs/12&#45;&gt;/outputs/13</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M334.303,-127C342.5813,-127 351.8217,-127 360.6405,-127\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"360.7163,-130.5001 370.7163,-127 360.7162,-123.5001 360.7163,-130.5001\"/>\n</g>\n<!-- /outputs/14 -->\n<g id=\"node12\" class=\"node\">\n<title>/outputs/14</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"569,-158 515,-158 515,-122 569,-122 569,-158\"/>\n<text text-anchor=\"start\" x=\"530\" y=\"-137\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Shape</text>\n</g>\n<!-- /outputs/13&#45;&gt;/outputs/14 -->\n<g id=\"edge13\" class=\"edge\">\n<title>/outputs/13&#45;&gt;/outputs/14</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M425.0853,-129.4452C447.4695,-131.466 479.4971,-134.3574 504.3725,-136.6031\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"504.2723,-140.1082 514.5466,-137.5216 504.9018,-133.1365 504.2723,-140.1082\"/>\n</g>\n<!-- /outputs/18 -->\n<g id=\"node16\" class=\"node\">\n<title>/outputs/18</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"923,-119 869,-119 869,-83 923,-83 923,-119\"/>\n<text text-anchor=\"start\" x=\"884\" y=\"-98\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Equal</text>\n</g>\n<!-- /outputs/13&#45;&gt;/outputs/18 -->\n<g id=\"edge14\" class=\"edge\">\n<title>/outputs/13&#45;&gt;/outputs/18</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M416.2141,-108.8468C441.9197,-85.2434 491.3798,-47 542,-47 542,-47 542,-47 758,-47 794.6362,-47 833.2145,-63.7089 860.2382,-78.5378\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"858.5143,-81.5837 868.9403,-83.4677 861.9648,-75.4932 858.5143,-81.5837\"/>\n</g>\n<!-- /outputs/19 -->\n<g id=\"node17\" class=\"node\">\n<title>/outputs/19</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1014,-187 960,-187 960,-151 1014,-151 1014,-187\"/>\n<text text-anchor=\"start\" x=\"974\" y=\"-166\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Where</text>\n</g>\n<!-- /outputs/13&#45;&gt;/outputs/19 -->\n<g id=\"edge15\" class=\"edge\">\n<title>/outputs/13&#45;&gt;/outputs/19</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M420.9706,-145.0348C448.221,-164.5762 495.7987,-193 542,-193 542,-193 542,-193 896,-193 914.131,-193 933.6421,-188.4905 949.9301,-183.3809\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"951.2317,-186.6365 959.6141,-180.1569 949.0205,-179.9949 951.2317,-186.6365\"/>\n</g>\n<!-- /outputs/15 -->\n<g id=\"node13\" class=\"node\">\n<title>/outputs/15</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"694,-165 606,-165 606,-129 694,-129 694,-165\"/>\n<text text-anchor=\"start\" x=\"614\" y=\"-144\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">ConstantOfShape</text>\n</g>\n<!-- /outputs/14&#45;&gt;/outputs/15 -->\n<g id=\"edge16\" class=\"edge\">\n<title>/outputs/14&#45;&gt;/outputs/15</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M569.253,-141.7664C577.4214,-142.2958 586.6822,-142.8961 595.9452,-143.4964\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"595.7402,-146.9904 605.9457,-144.1446 596.1931,-140.0051 595.7402,-146.9904\"/>\n</g>\n<!-- /outputs/17 -->\n<g id=\"node15\" class=\"node\">\n<title>/outputs/17</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"785,-119 731,-119 731,-83 785,-83 785,-119\"/>\n<text text-anchor=\"start\" x=\"749\" y=\"-98\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Mul</text>\n</g>\n<!-- /outputs/15&#45;&gt;/outputs/17 -->\n<g id=\"edge17\" class=\"edge\">\n<title>/outputs/15&#45;&gt;/outputs/17</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M692.3745,-128.9516C702.0236,-124.8418 712.1927,-120.5105 721.5687,-116.517\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"723.1687,-119.6399 730.9974,-112.5011 720.4256,-113.1997 723.1687,-119.6399\"/>\n</g>\n<!-- /outputs/15&#45;&gt;/outputs/19 -->\n<g id=\"edge18\" class=\"edge\">\n<title>/outputs/15&#45;&gt;/outputs/19</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M694.0569,-149.8761C760.845,-154.2362 886.5675,-162.4436 949.3432,-166.5417\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"949.3758,-170.0512 959.5826,-167.2101 949.8319,-163.0661 949.3758,-170.0512\"/>\n</g>\n<!-- /outputs/16 -->\n<g id=\"node14\" class=\"node\">\n<title>/outputs/16</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"677,-111 623,-111 623,-75 677,-75 677,-111\"/>\n<text text-anchor=\"start\" x=\"632\" y=\"-90\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Constant</text>\n</g>\n<!-- /outputs/16&#45;&gt;/outputs/17 -->\n<g id=\"edge19\" class=\"edge\">\n<title>/outputs/16&#45;&gt;/outputs/17</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M677.253,-95.0187C690.4273,-95.9946 706.4432,-97.181 720.7267,-98.239\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"720.5245,-101.7335 730.7558,-98.9819 721.0417,-94.7527 720.5245,-101.7335\"/>\n</g>\n<!-- /outputs/17&#45;&gt;/outputs/18 -->\n<g id=\"edge20\" class=\"edge\">\n<title>/outputs/17&#45;&gt;/outputs/18</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M785.2644,-101C806.2445,-101 835.4706,-101 858.6299,-101\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"858.7923,-104.5001 868.7923,-101 858.7922,-97.5001 858.7923,-104.5001\"/>\n</g>\n<!-- /outputs/18&#45;&gt;/outputs/19 -->\n<g id=\"edge21\" class=\"edge\">\n<title>/outputs/18&#45;&gt;/outputs/19</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M920.3863,-119.2228C930.8746,-127.0602 943.2861,-136.3347 954.5363,-144.7414\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"952.7563,-147.7805 962.8619,-150.9627 956.9464,-142.1731 952.7563,-147.7805\"/>\n</g>\n<!-- /outputs/20 -->\n<g id=\"node18\" class=\"node\">\n<title>/outputs/20</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1105,-206 1051,-206 1051,-170 1105,-170 1105,-206\"/>\n<text text-anchor=\"start\" x=\"1062\" y=\"-185\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Expand</text>\n</g>\n<!-- /outputs/19&#45;&gt;/outputs/20 -->\n<g id=\"edge22\" class=\"edge\">\n<title>/outputs/19&#45;&gt;/outputs/20</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1014.303,-174.7006C1022.5813,-176.4291 1031.8217,-178.3584 1040.6405,-180.1997\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1040.212,-183.6856 1050.7163,-182.3034 1041.6427,-176.8334 1040.212,-183.6856\"/>\n</g>\n<!-- /outputs/28 -->\n<g id=\"node26\" class=\"node\">\n<title>/outputs/28</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1253,-252 1199,-252 1199,-216 1253,-216 1253,-252\"/>\n<text text-anchor=\"start\" x=\"1209\" y=\"-231\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">MatMul</text>\n</g>\n<!-- /outputs/20&#45;&gt;/outputs/28 -->\n<g id=\"edge23\" class=\"edge\">\n<title>/outputs/20&#45;&gt;/outputs/28</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1105.3016,-191.2451C1126.432,-194.3055 1156.1796,-199.824 1181,-209 1183.8422,-210.0508 1186.721,-211.2617 1189.5767,-212.5721\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1188.0327,-215.7131 1198.5465,-217.0209 1191.143,-209.442 1188.0327,-215.7131\"/>\n<text text-anchor=\"middle\" x=\"1152\" y=\"-212\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">128x784x784</text>\n</g>\n<!-- /outputs/21 -->\n<g id=\"node19\" class=\"node\">\n<title>/outputs/21</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"145,-144 91,-144 91,-108 145,-108 145,-144\"/>\n<text text-anchor=\"start\" x=\"100\" y=\"-123\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Constant</text>\n</g>\n<!-- /outputs/24 -->\n<g id=\"node22\" class=\"node\">\n<title>/outputs/24</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"243,-144 182,-144 182,-108 243,-108 243,-144\"/>\n<text text-anchor=\"start\" x=\"190.5\" y=\"-123\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Unsqueeze</text>\n</g>\n<!-- /outputs/21&#45;&gt;/outputs/24 -->\n<g id=\"edge24\" class=\"edge\">\n<title>/outputs/21&#45;&gt;/outputs/24</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M145.0799,-126C153.3121,-126 162.5498,-126 171.4848,-126\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"171.7564,-129.5001 181.7563,-126 171.7563,-122.5001 171.7564,-129.5001\"/>\n</g>\n<!-- /outputs/22 -->\n<g id=\"node20\" class=\"node\">\n<title>/outputs/22</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"145,-360 91,-360 91,-324 145,-324 145,-360\"/>\n<text text-anchor=\"start\" x=\"100\" y=\"-339\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Constant</text>\n</g>\n<!-- /outputs/25 -->\n<g id=\"node23\" class=\"node\">\n<title>/outputs/25</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"243,-360 182,-360 182,-324 243,-324 243,-360\"/>\n<text text-anchor=\"start\" x=\"190.5\" y=\"-339\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Unsqueeze</text>\n</g>\n<!-- /outputs/22&#45;&gt;/outputs/25 -->\n<g id=\"edge25\" class=\"edge\">\n<title>/outputs/22&#45;&gt;/outputs/25</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M145.0799,-342C153.3121,-342 162.5498,-342 171.4848,-342\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"171.7564,-345.5001 181.7563,-342 171.7563,-338.5001 171.7564,-345.5001\"/>\n</g>\n<!-- /outputs/26 -->\n<g id=\"node24\" class=\"node\">\n<title>/outputs/26</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"334,-252 280,-252 280,-216 334,-216 334,-252\"/>\n<text text-anchor=\"start\" x=\"292\" y=\"-231\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Concat</text>\n</g>\n<!-- /outputs/23&#45;&gt;/outputs/26 -->\n<g id=\"edge26\" class=\"edge\">\n<title>/outputs/23&#45;&gt;/outputs/26</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M243.1814,-234C251.7033,-234 261.0485,-234 269.8957,-234\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"269.9735,-237.5001 279.9735,-234 269.9734,-230.5001 269.9735,-237.5001\"/>\n</g>\n<!-- /outputs/24&#45;&gt;/outputs/26 -->\n<g id=\"edge27\" class=\"edge\">\n<title>/outputs/24&#45;&gt;/outputs/26</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M233.9552,-144.1218C237.0751,-147.004 240.1819,-150.0197 243,-153 259.3581,-170.2993 275.973,-191.4563 288.1732,-207.8089\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"285.3723,-209.9079 294.126,-215.8765 291.0049,-205.7518 285.3723,-209.9079\"/>\n</g>\n<!-- /outputs/25&#45;&gt;/outputs/26 -->\n<g id=\"edge28\" class=\"edge\">\n<title>/outputs/25&#45;&gt;/outputs/26</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M234.4598,-323.7458C237.4443,-320.93 240.3784,-317.9683 243,-315 262.2591,-293.1939 261.4126,-283.3814 280,-261 280.3838,-260.5378 280.7743,-260.0746 281.1705,-259.6108\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"283.8041,-261.917 287.9382,-252.162 278.6231,-257.2098 283.8041,-261.917\"/>\n</g>\n<!-- /outputs/27 -->\n<g id=\"node25\" class=\"node\">\n<title>/outputs/27</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"497,-252 443,-252 443,-216 497,-216 497,-252\"/>\n<text text-anchor=\"start\" x=\"453\" y=\"-231\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Reshape</text>\n</g>\n<!-- /outputs/26&#45;&gt;/outputs/27 -->\n<g id=\"edge29\" class=\"edge\">\n<title>/outputs/26&#45;&gt;/outputs/27</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M334.2825,-234C361.279,-234 402.801,-234 432.8089,-234\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"432.9002,-237.5001 442.9002,-234 432.9001,-230.5001 432.9002,-237.5001\"/>\n</g>\n<!-- /outputs/27&#45;&gt;/outputs/28 -->\n<g id=\"edge30\" class=\"edge\">\n<title>/outputs/27&#45;&gt;/outputs/28</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M497.0681,-234C520.8835,-234 556.4931,-234 587.5,-234 587.5,-234 587.5,-234 1078,-234 1115.8095,-234 1159.0522,-234 1188.9898,-234\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1188.9935,-237.5001 1198.9934,-234 1188.9934,-230.5001 1188.9935,-237.5001\"/>\n<text text-anchor=\"middle\" x=\"827\" y=\"-237\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">128x784x1</text>\n</g>\n<!-- /outputs/33 -->\n<g id=\"node31\" class=\"node\">\n<title>/outputs/33</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1393,-284 1339,-284 1339,-248 1393,-248 1393,-284\"/>\n<text text-anchor=\"start\" x=\"1349\" y=\"-263\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Reshape</text>\n</g>\n<!-- /outputs/28&#45;&gt;/outputs/33 -->\n<g id=\"edge31\" class=\"edge\">\n<title>/outputs/28&#45;&gt;/outputs/33</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1253.3258,-240.2459C1274.8142,-245.1575 1304.9786,-252.0522 1328.6897,-257.4719\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1328.2147,-260.9535 1338.7432,-259.7699 1329.7745,-254.1295 1328.2147,-260.9535\"/>\n<text text-anchor=\"middle\" x=\"1295\" y=\"-257\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">128x784x1</text>\n</g>\n<!-- /outputs/29 -->\n<g id=\"node27\" class=\"node\">\n<title>/outputs/29</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"145,-414 91,-414 91,-378 145,-378 145,-414\"/>\n<text text-anchor=\"start\" x=\"100\" y=\"-393\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Constant</text>\n</g>\n<!-- /outputs/31 -->\n<g id=\"node29\" class=\"node\">\n<title>/outputs/31</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"243,-414 182,-414 182,-378 243,-378 243,-414\"/>\n<text text-anchor=\"start\" x=\"190.5\" y=\"-393\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Unsqueeze</text>\n</g>\n<!-- /outputs/29&#45;&gt;/outputs/31 -->\n<g id=\"edge32\" class=\"edge\">\n<title>/outputs/29&#45;&gt;/outputs/31</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M145.0799,-396C153.3121,-396 162.5498,-396 171.4848,-396\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"171.7564,-399.5001 181.7563,-396 171.7563,-392.5001 171.7564,-399.5001\"/>\n</g>\n<!-- /outputs/32 -->\n<g id=\"node30\" class=\"node\">\n<title>/outputs/32</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"334,-306 280,-306 280,-270 334,-270 334,-306\"/>\n<text text-anchor=\"start\" x=\"292\" y=\"-285\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Concat</text>\n</g>\n<!-- /outputs/30&#45;&gt;/outputs/32 -->\n<g id=\"edge33\" class=\"edge\">\n<title>/outputs/30&#45;&gt;/outputs/32</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M243.1814,-288C251.7033,-288 261.0485,-288 269.8957,-288\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"269.9735,-291.5001 279.9735,-288 269.9734,-284.5001 269.9735,-291.5001\"/>\n</g>\n<!-- /outputs/31&#45;&gt;/outputs/32 -->\n<g id=\"edge34\" class=\"edge\">\n<title>/outputs/31&#45;&gt;/outputs/32</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M233.9552,-377.8782C237.0751,-374.996 240.1819,-371.9803 243,-369 259.3581,-351.7007 275.973,-330.5437 288.1732,-314.1911\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"291.0049,-316.2482 294.126,-306.1235 285.3723,-312.0921 291.0049,-316.2482\"/>\n</g>\n<!-- /outputs/32&#45;&gt;/outputs/33 -->\n<g id=\"edge35\" class=\"edge\">\n<title>/outputs/32&#45;&gt;/outputs/33</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M334.063,-286.1873C352.1547,-285.1289 376.4931,-284 398,-284 398,-284 398,-284 1226,-284 1260.9847,-284 1300.4883,-278.4022 1328.591,-273.45\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1329.559,-276.8314 1338.7703,-271.5966 1328.305,-269.9446 1329.559,-276.8314\"/>\n</g>\n<!-- /outputs/35 -->\n<g id=\"node33\" class=\"node\">\n<title>/outputs/35</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1522,-247 1468,-247 1468,-211 1522,-211 1522,-247\"/>\n<text text-anchor=\"start\" x=\"1478\" y=\"-226\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">MatMul</text>\n</g>\n<!-- /outputs/33&#45;&gt;/outputs/35 -->\n<g id=\"edge36\" class=\"edge\">\n<title>/outputs/33&#45;&gt;/outputs/35</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1393.0426,-258.2436C1411.963,-252.8168 1437.4641,-245.5025 1458.2292,-239.5467\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1459.3455,-242.8677 1467.993,-236.7462 1457.4156,-236.139 1459.3455,-242.8677\"/>\n<text text-anchor=\"middle\" x=\"1431.5\" y=\"-254\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">128x784</text>\n</g>\n<!-- /outputs/34 -->\n<g id=\"node32\" class=\"node\">\n<title>/outputs/34</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1395,-230 1337,-230 1337,-194 1395,-194 1395,-230\"/>\n<text text-anchor=\"start\" x=\"1345\" y=\"-209\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Transpose</text>\n</g>\n<!-- /outputs/34&#45;&gt;/outputs/35 -->\n<g id=\"edge37\" class=\"edge\">\n<title>/outputs/34&#45;&gt;/outputs/35</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1395.1582,-215.8013C1411.3193,-217.9125 1431.7736,-220.5915 1450,-223 1452.5297,-223.3343 1455.1392,-223.68 1457.768,-224.0288\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1457.4997,-227.5239 1467.8739,-225.3727 1458.4225,-220.585 1457.4997,-227.5239\"/>\n<text text-anchor=\"middle\" x=\"1431.5\" y=\"-226\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">784x10</text>\n</g>\n<!-- /outputs/36 -->\n<g id=\"node34\" class=\"node\">\n<title>/outputs/36</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1644,-247 1590,-247 1590,-211 1644,-211 1644,-247\"/>\n<text text-anchor=\"start\" x=\"1600\" y=\"-226\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Softmax</text>\n</g>\n<!-- /outputs/35&#45;&gt;/outputs/36 -->\n<g id=\"edge38\" class=\"edge\">\n<title>/outputs/35&#45;&gt;/outputs/36</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1522.0758,-229C1539.0553,-229 1561.1767,-229 1579.7924,-229\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1580,-232.5001 1589.9999,-229 1579.9999,-225.5001 1580,-232.5001\"/>\n<text text-anchor=\"middle\" x=\"1556\" y=\"-232\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">128x10</text>\n</g>\n</g>\n</svg>\n"
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "54RiJNC_gzcy",
        "colab_type": "text"
      },
      "source": [
        "## Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z30YoxJKgzcz",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "5b77c41c-3e37-4672-886d-a1dada68c612"
      },
      "source": [
        "def compute_acc(model, data_loader, device):\n",
        "    correct_pred, num_examples = 0, 0\n",
        "    for features, targets in data_loader:\n",
        "        features = features.to(device)\n",
        "        targets = targets.to(device)\n",
        "        logits, probas = model(features)\n",
        "        _, predicted_labels = torch.max(probas, 1)\n",
        "        num_examples += targets.size(0)\n",
        "        correct_pred += (predicted_labels == targets).sum()\n",
        "    return correct_pred.float()/num_examples * 100\n",
        "    \n",
        "\n",
        "start_time = time.time()\n",
        "\n",
        "cost_list = []\n",
        "train_acc_list, valid_acc_list = [], []\n",
        "\n",
        "\n",
        "for epoch in range(NUM_EPOCHS):\n",
        "    \n",
        "    model.train()\n",
        "    for batch_idx, (features, targets) in enumerate(train_loader):\n",
        "        \n",
        "        features = features.to(DEVICE)\n",
        "        targets = targets.to(DEVICE)\n",
        "            \n",
        "        ### FORWARD AND BACK PROP\n",
        "        logits, probas = model(features)\n",
        "        cost = F.cross_entropy(logits, targets)\n",
        "        optimizer.zero_grad()\n",
        "        \n",
        "        cost.backward()\n",
        "        \n",
        "        ### UPDATE MODEL PARAMETERS\n",
        "        optimizer.step()\n",
        "        \n",
        "        #################################################\n",
        "        ### CODE ONLY FOR LOGGING BEYOND THIS POINT\n",
        "        ################################################\n",
        "        cost_list.append(cost.item())\n",
        "        if not batch_idx % 150:\n",
        "            print (f'Epoch: {epoch+1:03d}/{NUM_EPOCHS:03d} | '\n",
        "                   f'Batch {batch_idx:03d}/{len(train_loader):03d} |' \n",
        "                   f' Cost: {cost:.4f}')\n",
        "\n",
        "        \n",
        "\n",
        "    model.eval()\n",
        "    with torch.set_grad_enabled(False): # save memory during inference\n",
        "        \n",
        "        train_acc = compute_acc(model, train_loader, device=DEVICE)\n",
        "        valid_acc = compute_acc(model, valid_loader, device=DEVICE)\n",
        "        \n",
        "        print(f'Epoch: {epoch+1:03d}/{NUM_EPOCHS:03d}\\n'\n",
        "              f'Train ACC: {train_acc:.2f} | Validation ACC: {valid_acc:.2f}')\n",
        "        \n",
        "        train_acc_list.append(train_acc)\n",
        "        valid_acc_list.append(valid_acc)\n",
        "        \n",
        "    elapsed = (time.time() - start_time)/60\n",
        "    print(f'Time elapsed: {elapsed:.2f} min')\n",
        "  \n",
        "elapsed = (time.time() - start_time)/60\n",
        "print(f'Total Training Time: {elapsed:.2f} min')"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch: 001/020 | Batch 000/461 | Cost: 2.2677\n",
            "Epoch: 001/020 | Batch 150/461 | Cost: 0.8999\n",
            "Epoch: 001/020 | Batch 300/461 | Cost: 0.6701\n",
            "Epoch: 001/020 | Batch 450/461 | Cost: 0.4905\n",
            "Epoch: 001/020\n",
            "Train ACC: 87.02 | Validation ACC: 92.40\n",
            "Time elapsed: 0.64 min\n",
            "Epoch: 002/020 | Batch 000/461 | Cost: 0.5868\n",
            "Epoch: 002/020 | Batch 150/461 | Cost: 0.4526\n",
            "Epoch: 002/020 | Batch 300/461 | Cost: 0.4192\n",
            "Epoch: 002/020 | Batch 450/461 | Cost: 0.3647\n",
            "Epoch: 002/020\n",
            "Train ACC: 88.25 | Validation ACC: 92.60\n",
            "Time elapsed: 1.27 min\n",
            "Epoch: 003/020 | Batch 000/461 | Cost: 0.4316\n",
            "Epoch: 003/020 | Batch 150/461 | Cost: 0.4165\n",
            "Epoch: 003/020 | Batch 300/461 | Cost: 0.4130\n",
            "Epoch: 003/020 | Batch 450/461 | Cost: 0.3991\n",
            "Epoch: 003/020\n",
            "Train ACC: 88.80 | Validation ACC: 93.30\n",
            "Time elapsed: 1.90 min\n",
            "Epoch: 004/020 | Batch 000/461 | Cost: 0.3537\n",
            "Epoch: 004/020 | Batch 150/461 | Cost: 0.3460\n",
            "Epoch: 004/020 | Batch 300/461 | Cost: 0.4011\n",
            "Epoch: 004/020 | Batch 450/461 | Cost: 0.4666\n",
            "Epoch: 004/020\n",
            "Train ACC: 89.34 | Validation ACC: 93.40\n",
            "Time elapsed: 2.53 min\n",
            "Epoch: 005/020 | Batch 000/461 | Cost: 0.4523\n",
            "Epoch: 005/020 | Batch 150/461 | Cost: 0.4006\n",
            "Epoch: 005/020 | Batch 300/461 | Cost: 0.4396\n",
            "Epoch: 005/020 | Batch 450/461 | Cost: 0.4509\n",
            "Epoch: 005/020\n",
            "Train ACC: 89.65 | Validation ACC: 93.40\n",
            "Time elapsed: 3.16 min\n",
            "Epoch: 006/020 | Batch 000/461 | Cost: 0.3381\n",
            "Epoch: 006/020 | Batch 150/461 | Cost: 0.3627\n",
            "Epoch: 006/020 | Batch 300/461 | Cost: 0.2736\n",
            "Epoch: 006/020 | Batch 450/461 | Cost: 0.3932\n",
            "Epoch: 006/020\n",
            "Train ACC: 89.85 | Validation ACC: 93.50\n",
            "Time elapsed: 3.79 min\n",
            "Epoch: 007/020 | Batch 000/461 | Cost: 0.4984\n",
            "Epoch: 007/020 | Batch 150/461 | Cost: 0.3718\n",
            "Epoch: 007/020 | Batch 300/461 | Cost: 0.2903\n",
            "Epoch: 007/020 | Batch 450/461 | Cost: 0.4040\n",
            "Epoch: 007/020\n",
            "Train ACC: 90.02 | Validation ACC: 93.50\n",
            "Time elapsed: 4.42 min\n",
            "Epoch: 008/020 | Batch 000/461 | Cost: 0.5250\n",
            "Epoch: 008/020 | Batch 150/461 | Cost: 0.3481\n",
            "Epoch: 008/020 | Batch 300/461 | Cost: 0.3838\n",
            "Epoch: 008/020 | Batch 450/461 | Cost: 0.4789\n",
            "Epoch: 008/020\n",
            "Train ACC: 90.14 | Validation ACC: 93.90\n",
            "Time elapsed: 5.05 min\n",
            "Epoch: 009/020 | Batch 000/461 | Cost: 0.3028\n",
            "Epoch: 009/020 | Batch 150/461 | Cost: 0.3982\n",
            "Epoch: 009/020 | Batch 300/461 | Cost: 0.4042\n",
            "Epoch: 009/020 | Batch 450/461 | Cost: 0.5471\n",
            "Epoch: 009/020\n",
            "Train ACC: 90.26 | Validation ACC: 93.90\n",
            "Time elapsed: 5.68 min\n",
            "Epoch: 010/020 | Batch 000/461 | Cost: 0.2279\n",
            "Epoch: 010/020 | Batch 150/461 | Cost: 0.2992\n",
            "Epoch: 010/020 | Batch 300/461 | Cost: 0.4507\n",
            "Epoch: 010/020 | Batch 450/461 | Cost: 0.2165\n",
            "Epoch: 010/020\n",
            "Train ACC: 90.40 | Validation ACC: 93.90\n",
            "Time elapsed: 6.31 min\n",
            "Epoch: 011/020 | Batch 000/461 | Cost: 0.5089\n",
            "Epoch: 011/020 | Batch 150/461 | Cost: 0.2480\n",
            "Epoch: 011/020 | Batch 300/461 | Cost: 0.3782\n",
            "Epoch: 011/020 | Batch 450/461 | Cost: 0.3228\n",
            "Epoch: 011/020\n",
            "Train ACC: 90.47 | Validation ACC: 93.40\n",
            "Time elapsed: 6.94 min\n",
            "Epoch: 012/020 | Batch 000/461 | Cost: 0.2597\n",
            "Epoch: 012/020 | Batch 150/461 | Cost: 0.2955\n",
            "Epoch: 012/020 | Batch 300/461 | Cost: 0.2243\n",
            "Epoch: 012/020 | Batch 450/461 | Cost: 0.2967\n",
            "Epoch: 012/020\n",
            "Train ACC: 90.58 | Validation ACC: 93.60\n",
            "Time elapsed: 7.57 min\n",
            "Epoch: 013/020 | Batch 000/461 | Cost: 0.3367\n",
            "Epoch: 013/020 | Batch 150/461 | Cost: 0.3696\n",
            "Epoch: 013/020 | Batch 300/461 | Cost: 0.2744\n",
            "Epoch: 013/020 | Batch 450/461 | Cost: 0.4097\n",
            "Epoch: 013/020\n",
            "Train ACC: 90.65 | Validation ACC: 93.80\n",
            "Time elapsed: 8.20 min\n",
            "Epoch: 014/020 | Batch 000/461 | Cost: 0.2629\n",
            "Epoch: 014/020 | Batch 150/461 | Cost: 0.3282\n",
            "Epoch: 014/020 | Batch 300/461 | Cost: 0.2407\n",
            "Epoch: 014/020 | Batch 450/461 | Cost: 0.2714\n",
            "Epoch: 014/020\n",
            "Train ACC: 90.66 | Validation ACC: 93.80\n",
            "Time elapsed: 8.84 min\n",
            "Epoch: 015/020 | Batch 000/461 | Cost: 0.2497\n",
            "Epoch: 015/020 | Batch 150/461 | Cost: 0.3774\n",
            "Epoch: 015/020 | Batch 300/461 | Cost: 0.3405\n",
            "Epoch: 015/020 | Batch 450/461 | Cost: 0.4727\n",
            "Epoch: 015/020\n",
            "Train ACC: 90.81 | Validation ACC: 93.90\n",
            "Time elapsed: 9.47 min\n",
            "Epoch: 016/020 | Batch 000/461 | Cost: 0.4100\n",
            "Epoch: 016/020 | Batch 150/461 | Cost: 0.3284\n",
            "Epoch: 016/020 | Batch 300/461 | Cost: 0.3974\n",
            "Epoch: 016/020 | Batch 450/461 | Cost: 0.2978\n",
            "Epoch: 016/020\n",
            "Train ACC: 90.86 | Validation ACC: 93.90\n",
            "Time elapsed: 10.10 min\n",
            "Epoch: 017/020 | Batch 000/461 | Cost: 0.2101\n",
            "Epoch: 017/020 | Batch 150/461 | Cost: 0.3024\n",
            "Epoch: 017/020 | Batch 300/461 | Cost: 0.2714\n",
            "Epoch: 017/020 | Batch 450/461 | Cost: 0.2259\n",
            "Epoch: 017/020\n",
            "Train ACC: 90.91 | Validation ACC: 93.90\n",
            "Time elapsed: 10.73 min\n",
            "Epoch: 018/020 | Batch 000/461 | Cost: 0.3154\n",
            "Epoch: 018/020 | Batch 150/461 | Cost: 0.2534\n",
            "Epoch: 018/020 | Batch 300/461 | Cost: 0.3008\n",
            "Epoch: 018/020 | Batch 450/461 | Cost: 0.2815\n",
            "Epoch: 018/020\n",
            "Train ACC: 90.98 | Validation ACC: 93.90\n",
            "Time elapsed: 11.37 min\n",
            "Epoch: 019/020 | Batch 000/461 | Cost: 0.2850\n",
            "Epoch: 019/020 | Batch 150/461 | Cost: 0.2086\n",
            "Epoch: 019/020 | Batch 300/461 | Cost: 0.4104\n",
            "Epoch: 019/020 | Batch 450/461 | Cost: 0.2749\n",
            "Epoch: 019/020\n",
            "Train ACC: 90.94 | Validation ACC: 94.00\n",
            "Time elapsed: 12.00 min\n",
            "Epoch: 020/020 | Batch 000/461 | Cost: 0.4211\n",
            "Epoch: 020/020 | Batch 150/461 | Cost: 0.2129\n",
            "Epoch: 020/020 | Batch 300/461 | Cost: 0.2256\n",
            "Epoch: 020/020 | Batch 450/461 | Cost: 0.5096\n",
            "Epoch: 020/020\n",
            "Train ACC: 91.02 | Validation ACC: 94.30\n",
            "Time elapsed: 12.63 min\n",
            "Total Training Time: 12.63 min\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6t7_i5yUgzc1",
        "colab_type": "text"
      },
      "source": [
        "## Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yZ_O6Sukgzc1",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        },
        "outputId": "35a01150-f0f0-4263-b129-0686ffd16711"
      },
      "source": [
        "plt.plot(cost_list, label='Minibatch cost')\n",
        "plt.plot(np.convolve(cost_list, \n",
        "                     np.ones(200,)/200, mode='valid'), \n",
        "         label='Running average')\n",
        "\n",
        "plt.ylabel('Cross Entropy')\n",
        "plt.xlabel('Iteration')\n",
        "plt.legend()\n",
        "plt.show()"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3hTZfvA8e+ddLH3XmXLKiBTcCDIEBS3OH5uxQmO14FbcKGv+rpw4J6Ie6EoqMgG2XtKgbKhlBboTJ7fHzlJ0zZJ05GmJffnunqRnPnkkJz7PFuMMSillIpctnAnQCmlVHhpIFBKqQingUAppSKcBgKllIpwGgiUUirCRYU7AUVVt25dEx8fH+5kKKVUhbJ06dKDxph6vtZVuEAQHx/PkiVLwp0MpZSqUERku791WjSklFIRTgOBUkpFOA0ESikV4SpcHYFSqmSys7NJSkoiIyMj3ElRIRAXF0fTpk2Jjo4Oeh8NBEpFmKSkJKpVq0Z8fDwiEu7kqFJkjOHQoUMkJSXRsmXLoPfToiGlIkxGRgZ16tTRIHACEhHq1KlT5NyeBgKlIpAGgRNXcf5vIyYQbNybxou/b+TQ0cxwJ0UppcqViAkEW/Yf5bU/t3DwaFa4k6JUxBMR/u///s/zPicnh3r16nHOOecA8OOPPzJx4sSAx9i9ezcXX3wxAB9++CF33HFHkdLwzDPPFLrNtddey9dff12k4xbHihUr+OWXX0J+Hn8iJhBE2V3ZpWyHM8wpUUpVqVKFNWvWkJ6eDsCMGTNo0qSJZ/3IkSMZN25cwGM0bty4RDfpYAJBWdFAUEairUCQ49QZ2ZQqD4YPH860adMAmDJlCpdffrlnnfcT/rXXXsvYsWPp168frVq18tz8ExMT6dy5s2efnTt3MmDAANq2bcv48eM9y88//3x69OhBp06dmDx5MgDjxo0jPT2dbt26ceWVVwLw8ccfk5CQQNeuXbnqqqs8+8+ePbvAufPztW9iYiIDBw4kISGBQYMGsWPHDgC++uorOnfuTNeuXTn99NPJysriscceY+rUqXTr1o2pU6eW7MIWQ8Q0H42yuWKe5giUyjX+p7Ws251aqsfs2Lg6j5/bqdDtLrvsMiZMmMA555zDqlWruP7665kzZ47Pbffs2cPcuXPZsGEDI0eO9BQJeVu8eDFr1qyhcuXK9OrVixEjRtCzZ0/ef/99ateuTXp6Or169eKiiy5i4sSJvP7666xYsQKAtWvX8tRTTzF//nzq1q1LcnJy0Of2t++YMWO45ppruOaaa3j//fcZO3Ys33//PRMmTOC3336jSZMmpKSkEBMTw4QJE1iyZAmvv/560Ne5NEVMjsBuc+UInJojUKpcSEhIIDExkSlTpjB8+PCA255//vnYbDY6duzIvn37fG4zePBg6tSpQ6VKlbjwwguZO3cuAK+++ipdu3alb9++7Ny5k82bNxfY988//+SSSy6hbt26ANSuXTvoc/vbd8GCBVxxxRUAXHXVVZ709O/fn2uvvZZ33nkHh8MR8HOXlYjJEdisJlUaB5TKFcyTeyiNHDmSe++9l1mzZnHo0CG/28XGxnpeG+P7R5y/2aSIMGvWLGbOnMmCBQuoXLkyAwYMKHIb+2DOXRRvvfUWixYtYtq0afTo0YOlS5eW+JglFTE5AitDUCr/kUqp0nH99dfz+OOP06VLlxIfa8aMGSQnJ5Oens73339P//79OXLkCLVq1aJy5cps2LCBhQsXeraPjo4mOzsbgIEDB/LVV195gpF30VBh/O3br18/vvjiCwA+++wzTjvtNAC2bt1Knz59mDBhAvXq1WPnzp1Uq1aNtLS0El+D4oqcQGBFAocGAqXKjaZNmzJ27NhSOVbv3r256KKLSEhI4KKLLqJnz54MGzaMnJwcOnTowLhx4+jbt69n+9GjR5OQkMCVV15Jp06dePjhhznjjDPo2rUr99xzT9Dn9bfva6+9xgcffEBCQgKffPIJr7zyCgD33XcfXbp0oXPnzvTr14+uXbty5plnsm7durBVFktFe0Lu2bOnKc7ENEu3H+aiN+fz0fW9OaOdz0l6lIoI69evp0OHDuFOhgohX//HIrLUGNPT1/aRkyOwioacFSzwKaVUqEVQIHBFgoqWA1JKqVCLuECg3QiUUiqvyAkE1ifVoiGllMorcgKBFg0ppZRPERcItEOZUkrlFUGBwPWvQyOBUmFnt9vp1q0bnTt35txzzyUlJaXUz/HWW2/x8ccfl/pxT0SREwjcYw1p0ZBSYVepUiVWrFjBmjVrqF27NpMmTSr1c9xyyy1cffXVpX7c0pKTkxPuJHhETiDw1BGEOSFKqTxOOeUUdu3aBcCAAQNwdxg9ePAg8fHxgGtY6gsvvJBhw4bRtm1b7r//fs/+VatW5eGHH/YMLOceGO6JJ57ghRde8Bz3gQceoHfv3rRr184zyunx48e59NJL6dixIxdccAF9+vTBV4fVCRMm0KtXLzp37szo0aMxxrBhwwZ69+7t2SYxMdEzVMbSpUs544wz6NGjB0OHDmXPnj2edNx111307NmTV155hZ9++ok+ffrQvXt3zjrrLE/aDxw4wODBg+nUqRM33ngjLVq04ODBgwB8+umn9O7dm27dunHzzTeXysB1ETTonOtfLRpSysuv42Dv6tI9ZsMucHbg2cXcHA4Hf/zxBzfccEOh265YsYLly5cTGxtL+/btGTNmDM2aNePYsWP07duXp59+mvvvv5933nmHRx55pMD+OTk5LF68mF9++YXx48czc+ZM3njjDWrVqsW6detYs2YN3bp183nuO+64g8ceewxwjST6888/c+6555KVlcW2bdto2bIlU6dOZdSoUWRnZzNmzBh++OEH6tWrx9SpU3n44Yd5//33AcjKyvIEm8OHD7Nw4UJEhHfffZfnn3+eF198kfHjxzNw4EAefPBBpk+fznvvvQe4egxPnTqVefPmER0dzW233cZnn31W4pxPBAUCLRpSqrxwTwqza9cuOnTowODBgwvdZ9CgQdSoUQOAjh07sn37dpo1a0ZMTIxnissePXowY8YMn/tfeOGFnm0SExMBmDt3LnfeeScAnTt3JiEhwee+f/31F88//zzHjx8nOTmZTp06ce6553LppZcydepUxo0bx9SpU5k6dSobN25kzZo1ns/kcDho1KiR51ijRo3yvE5KSmLUqFHs2bOHrKwsWrZs6UnXd999B8CwYcOoVasWAH/88QdLly6lV69enutYv379Qq9dYSInENi0aEipAoJ8ci9t7jqC48ePM3ToUCZNmsTYsWOJiorC6XT1+sw/XLT3cNB2u91Txh4dHe0Zgtp7eX7u/QNt40tGRga33XYbS5YsoVmzZjzxxBOetI0aNYpLLrmECy+8EBGhbdu2rF69mk6dOrFgwQKfx6tSpYrn9ZgxY7jnnnsYOXIks2bN4oknngiYFmMM11xzDc8++2zQ6Q9GBNURuP7VHIFS5UflypV59dVXefHFF8nJySE+Pt4zPn9ZTBrfv39/vvzySwDWrVvH6tUFi8ncN/26dety9OjRPOlq3bo1drudJ5980vOk3759ew4cOOAJBNnZ2axdu9bn+Y8cOeKZq/mjjz7yma7ff/+dw4cPA65c0ddff83+/fsB15DX27dvL/4FsIQsEIhIMxH5S0TWichaEbnTxzYiIq+KyBYRWSUiJ4cqPZ4hJjQQKFWudO/enYSEBKZMmcK9997Lm2++Sffu3T2Vo6F02223ceDAATp27MgjjzxCp06dPMVPbjVr1uSmm26ic+fODB061FMs4zZq1Cg+/fRTLr30UgBiYmL4+uuveeCBB+jatSvdunVj/vz5Ps//xBNPcMkll9CjRw/PDGcAjz/+OL///judO3fmq6++omHDhlSrVo2OHTvy1FNPMWTIEBISEhg8eLCnIrpEjDEh+QMaASdbr6sBm4CO+bYZDvwKCNAXWFTYcXv06GGKY39qhmnxwM/m4/nbirW/UieKdevWhTsJ5UZOTo5JT083xhizZcsWEx8fbzIzM8OcKmMyMjJMdna2McaY+fPnm65duxZpf1//x8AS4+e+GrI6AmPMHmCP9TpNRNYDTYB1XpudB3xsJXKhiNQUkUbWvqXKPWexthpSSrkdP36cM888k+zsbIwxvPHGG8TExIQ7WezYsYNLL70Up9NJTEwM77zzTkjPVyaVxSISD3QHFuVb1QTY6fU+yVqWJxCIyGhgNEDz5s2LlQa7p2ioWLsrpU5A1apV89lvINzatm3L8uXLy+x8Ia8sFpGqwDfAXcaY1OIcwxgz2RjT0xjTs1694s0u5h591GgdgVL6OziBFef/NqSBQESicQWBz4wx3/rYZBfQzOt9U2tZqcudj0B/ACqyxcXFcejQIQ0GJyBjDIcOHSIuLq5I+4WsaEhcDXvfA9YbY17ys9mPwB0i8gXQBzgSivoB8Koj0C+/inBNmzYlKSmJAwcOhDspKgTi4uJo2rRpkfYJZR1Bf+AqYLWIrLCWPQQ0BzDGvAX8gqvl0BbgOHBdqBLj6VmsOQIV4aKjoz09WJWCEAYCY8xcXM1CA21jgNtDlQZvua2GyuJsSilVcURcz2ItGlJKqbwiJhCICCLaWkIppfKLmEAArr4E2mpIKaXyiqhAYLOJFg0ppVQ+ERUI7CLaakgppfKJrEBgE201pJRS+URUILCJzkeglFL5RVYgsIkGAqWUyieiAoG2GlJKqYIiKhBojkAppQqKqECgOQKllCoosgKBTdA4oJRSeUVUIBDR0UeVUiq/iAoERzNzOHQsK9zJUEqpciWiAkHK8Wz+3qSTcSillLeICgRKKaUK0kCglFIRTgOBUkpFOA0ESikV4TQQKKVUhNNAoJRSES6iAsHwLg2pXSUm3MlQSqlyJaICQfW4aKLtEu5kKKVUuRJRgcBuE3IcOsSEUkp5i6hAEGUTcnSsIaWUyiOyAoHdpsNQK6VUPpEVCGxCts5er5RSeURWILDrxDRKKZVfRAUCu81GjtNgdLpKpZTyiKhAEG1zNR3VXIFSSuWKqEBgt/oQaMshpZTKFVGBIMqmgUAppfKLqEAwc91+AA6kZYY5JUopVX5EVCBYnJgMwOZ9aWFOiVJKlR8RFQjuG9oegHYNqoU5JUopVX4UGghEZIyI1CqLxIRa45pxAGgNgVJK5QomR9AA+EdEvhSRYSJSYYfvtFlJd2o/AqWU8ig0EBhjHgHaAu8B1wKbReQZEWkdaD8ReV9E9ovIGj/rB4jIERFZYf09Voz0F4k7hmkcUEqpXEHVERhXV9y91l8OUAv4WkSeD7Dbh8CwQg49xxjTzfqbEExaSsJqPao9i5VSyktUYRuIyJ3A1cBB4F3gPmNMtojYgM3A/b72M8bMFpH40ktqyeUWDYU5IUopVY4UGgiA2sCFxpjt3guNMU4ROaeE5z9FRFYCu4F7jTFrfW0kIqOB0QDNmzcv9sncOQKtI1BKqVzB1BE8DtQRkbFWC6KTvdatL8G5lwEtjDFdgdeA7wOkYbIxpqcxpme9evWKfULRymKllCogmOajjwIfAXWAusAHIvJISU9sjEk1xhy1Xv8CRItI3ZIeNxD3NJWHjmaF8jRKKVWhBFNZ/H9AL2PM41buoC9wVUlPLCIN3U1RRaS3lZZDJT1uINPX7gVg9CdLQnkapZSqUIKpI9gNxAEZ1vtYYFdhO4nIFGAAUFdEkoDHgWgAY8xbwMXArSKSA6QDl5kQN+c5aI0xlJGts5QppZRbMIHgCLBWRGbg6pQ7GFgsIq8CGGPG+trJGHN5oIMaY14HXi9ackvGbquwfeGUUipkggkE31l/brNCk5TQ00CglFIFFRoIjDEfiUgM0M5atNEYkx3aZIVGlAYCpZQqIJgOZQNwtRpKBARoJiLXGGNmhzZppa9WlZhwJ0EppcqdYIqGXgSGGGM2AohIO2AK0COUCQuFhtXjwp0EpZQqd4JpPhrtDgIAxphNWK1/KpqKO26qUkqFTjCBYKmIvGuNFjpARN4BKmRD/OFdGoU7CUopVe4EUzR0C3A74G4mOgd4I2QpCqEOjaqHOwlKKVXuBAwEImIHVhpjTgJeKpskKaWUKksBi4aMMQ5go4gUf8hPpZRS5VowRUO1cPUsXgwccy80xowMWaqUUkqVmWACwaMhT0UZSmhag/2pmeFOhlJKlRvBBILhxpgHvBeIyHPA36FJUmg5jWFvakbhGyqlVIQIpvnoYB/Lzi7thJSVNbtSw50EpZQqV/wGAhG5VURWA+1FZJXX3zZgddklMTS27D8a7iQopVS5EChH8DlwLvCj9a/7r4cx5soySFtI/bxqd7iToJRS5YLfOgJjzBFccxFcbvUnaGBtX1VEqhpjdpRRGpVSSoVQMKOP3gE8AewD3FN7GSAhdMlSSilVVoJpNXQX0N4YE9L5hMvazuT0cCdBKaXKhWBaDe3EVUR0Qkk5nhXuJCilVLkQTI7gX2CWiEwDPD2xjDEVeuyhmpV1khqllILgAsEO6y/G+jshnNG+XriToJRS5UIwcxaPz79MRIIJIOWazlGjlFIugTqUzfV6/Um+1YtDlqIy8u7cbeFOglJKlQuBKoureL3unG9dhX2g7t68JgArd6aEOSVKKVU+BAoExs9rX+8rjAfP7hDuJCilVLkSqKy/pohcgCtY1BSRC63lAtQIecpCxFZh8zJKKRUagQLB38BIr9fneq2bHbIUhZhoIFBKqTwCjTV0XVkmpKyYCluopZRSoRFMz+ITisYBpZTKq8L3BwiaMZCTgXE4wp0SpZQqVyInR7DmG3i6IbGpuf0Hvl++K4wJUkqp8qHQQCAil4hINev1IyLyrYicHPqklbLYagDYs3NnJvtjw/5wpUYppcqNYHIEjxpj0kTkVOAs4D3gzdAmKwSsQFA/JnfU0cxsLSZSSqlgAoH7bjkCmGyMmUZFHHwutjqQNxBoxbFSSgUXCHaJyNvAKOAXEYkNcr/yxcoRkJnmWaRNSZVSKrgb+qXAb8BQY0wKUBu4L6SpCgUfgUAppVRwzUcbAdOMMZkiMgDXXMUfhzRVoeAjECzbcThMiVFKqfIjmBzBN4BDRNoAk4FmwOeF7SQi74vIfhFZ42e9iMirIrJFRFaFvCWSPRqiKkFmqmdR8jGdrlIppYIJBE5jTA5wIfCaMeY+XLmEwnwIDAuw/mygrfU3mrJoiRRbLU8gUEopFVwgyBaRy4GrgZ+tZdGF7WSMmQ0kB9jkPOBj47IQ1winwQSY4outpnUESimVTzCB4DrgFOBpY8w2EWkJ5J+xrDiaADu93idZy0JHA4FSShVQaCAwxqwD7gVWi0hnIMkY81zIU+ZFREaLyBIRWXLgwIHiH0gDgVJKFRDMEBMDgM3AJOANYJOInF4K596Fq+LZram1rABjzGRjTE9jTM969eoV/4yVakK6thRSSilvwRQNvQgMMcacYYw5HRgK/K8Uzv0jcLXVeqgvcMQYs6cUjutf5Tpw/FCeRbM26nhDSqnIFkwgiDbGbHS/McZsIojKYhGZAiwA2otIkojcICK3iMgt1ia/AP8CW4B3gNuKnPqiqlwXjicjOD2Lrv3gn5CfVimlyrNgOpQtFZF3gU+t91cCSwrbyRhzeSHrDXB7EOcvPVXqgnFwY89avLPkSJmeWimlyqtgAsEtuG7YY633c3DVFVQ8lesCUNNoXwKllHILGAhExA6sNMacBLxUNkkKocq1AahjOwpUCW9alFKqnAhYR2CMcQAbRaR5GaUntKq4cgSVs7XlkFJKuQVTNFQLWCsii4Fj7oXGmJEhS1WoWEVDlXMO42qt6pKakU31uELrv5VS6oQUTCB4NOSpKCtWjqBR1LE8ixOe+J1XL+/OyK6Nw5EqpZQKK7+BwBpttIEx5u98y08FQtveP1SiYiG2Oh2rZxRY9dasrRoIlFIRKVAdwcuAr+Y1R6x1FVP1JsgRnx2YlVIqIgUKBA2MMavzL7SWxYcsRaFWsxkc2VFgsUgY0qKUUuVAoEBQM8C6SqWdkDJToxmk7Cx8O6WUihCBAsESEbkp/0IRuRFYGrokhVjNZpCRQhXS8yzWHIFSKlIFajV0F/CdiFxJ7o2/JxADXBDqhIVMDdeAp03kIJtMs0I2VkqpE5/fQGCM2Qf0E5Ezgc7W4mnGmD/LJGWhUtPVNy5/IFizS4edUEpFpmAmpvnLGPOa9VexgwDkyRHkN3/rQYb+bzaZOY6yTpVSSoVNMMNQn1iqNgB7LP3rHC2w6pHv17BxXxo7k4+HIWFKKRUekRcIbDaofxIdZHuBVQfTMq1XWnOslIockRcIABp1pUXWFsDkWZyakROe9CilVBhFbCCQ9GSaULCeAOCX1RVzBA2llCqOCA0E3QHobEv0ufqlGZvKMDFKKRVekRkIGnQEsTOg+m6/m2jLIaVUpIjMQBBdCeqdRHvzr99NjmW6AsFva/eSePCY3+2UUqqii8xAANCoK+2c/gOBMYbth45x8ydLGfDCrLJLl1JKlbGIDgRVsw/RgGSfq2/9dBlv/e0/ULw7518e+HpVqFJXQEa2g+NZ2qpJKVX6IjcQNOsNQB/bBp+rFycmM2Vx7nDV09fktiTauDeNp6atZ+qSshvF9LTn/6LjY7+V2fmUUpEjcgNBo66Y2Bqcaisw5YJPm/bl9kQe+vLsUKXKrwOezm4nvulr9vDe3G3hToZfb8zawrwtvpseK1URRW4gsNmRtmcx0L4cG85CN1+y/XAZJEoB3PLpMp78eV24k+HX89M3cuW7i8KdDKVKTeQGAoD2w6krqfSxrS9009mbDvhcHj9uGn/7WaeUUhVBZAeCk0aQaipzpf2PEh3mt7V7SyU5xpjCN1JKVTiZOY5y3TcpsgNBdCVsPa/lbNsiWkjxb+alcQP/bnkSLR/8RUc+jRArd6awLzUj3MlQZeTkCTNo/8h0bv9sWbiT4lNkBwLg6Mk3k0MUY6O+K3Tb+HHTiB83rcByY2Dp9sP8sGIXAHM3H2TaqqKNV/TjClcv50d/WEOXx38rtVyG8m1/agbPTd+A0xmeXNh5k+Zxxn//Csu5Vdk7luXKDUwrp+OYRXwgcFRpwAeOoVxkn0OCbC3WMXYePs5Fb87nzi9WsHJnCv/33iJu/3wZD30XXIskb7M2HiAtM4eJv/pu1loSUxbvYPE23/0m8svIdrB295FST0N5cd/Xq3hz1lYWJwZ3PUIhI7vwRgqBPPHjWuZr6yVVCiI+EMRG2Xgt5wIOmBo8Ff0+dopejjdvyyHP6/MmzfO8/nzRDl+bByUU9QUPfruaS99eENS2D327mhGvzuXtv7eWeloW/XuI34uQ4zHGcP/XK1n076HCNw6Su7zWWYHrZT6cn8gV2npJlYKIDwR1q8bSr2M8j2dfQ4JtGzfYfwnZuVKOZ3H/1ytJt7KJ2Q4n87ccJOlw+asXWLbD1Vz22V835Al03v49cJSxU5aT7Sjak+2oyQsZ/cnSoLd3GvhySRKXv7OwSOepaLbsP0pqRnaJj5OZ42DgC7NO+NZsw1+ZQ/tHfg13Mk4IER8IAB4Y1p5fnH2Y4TiZu6K+pamU3g9ozuYDPDd9A6PeXsDLMzfz5ZIkpv7jyim88PtGrnh3Eac+9xf5i6rD/Zxqt+XO0nY0M3doi7Ne+tszTPf9X6/ix5W7WbEzpczTl9/KnSlkZAefm8ufEViSmMz1H/6DowzqDPx1Djzrpb8Z9XbJg93ulAz+PXiMGz/6h/hx05i+pnzUNxljSDmeVWrHW7cnlcyc3IcQYwx9npnJl/+4evyvSkrhrb+LV9wbaTQQAG3qVwOEx7Kvo7Jk8lPMw1QhvVSOfdV7i3lz1lYWbUv2FLHst24EG/akebbL//TmvlEZY7hn6grP8pdn5s6V4HQaViX5vgkfSMvkxo/+4Uh68E+Y7875l/hx09iVkp4nEHjbsv8or/6xGQCbuLYJ1c0z8eAxXv1jc6FFU3uOpHPepHk+62RSM7I5ctz/NRBrWtIxU5bz54b97LVa8gx7eTbP/FJ4/5Li6PX0TL/r1u9JLbXzZDtc1+3bZUmldsySeH9eIt0mzGDHodDkgJ0G9qVmMu7bVRxIy2Tk6/Py1LXtS80ocY5rSWJySBsY/OfLlZz72tyQHd8fDQSWmfecwR7q8ED2TdSSo7wX8wLRlO4gb+4A8MasrUz6a0vAbd031yyHk2+X7/Isf3nmZs/r9+dtY+Tr8/hw3jaycpxc8MY8Flrl6G/O2srM9fv5qgjjIT01zXXjW77jsOcm75aakV2gJZR7k2DL2f/z5Uoe/Db4gfoGvDCLl2ZsYl8hw2ukWVOMrkpyVW7vTD7O29aTYNfxv9N1wu8F9smfYvfndQedDXvTmDzb/6CD5Vl5nXH7j/X7AFfjilAy4PN71ueZPzjrxb+LfdxZG/dz8VsLeH9eyYY/efZX/w8Y3yxLYvWusm+koYHA0qZ+VUaf3oqpjjN5KPsG+trWMzH6nVI9xzqvp73X/9zCsUz/gcZdzCE+ftbvz91GyvEs1ls5iid+Wke7R35l+Y4ULpu8kCPHs1lqlfH/GqBY4NDRTE8alu/IHULD4TQFAsF/vlzJ7Z/nbQOde/MseOw/N+zjzw378iz7ZlkSUxYXfaC+wp7A3Cl138Svem8Rz/66gSd/XuczbXn2zXd5QzGmU1aOs1SOe8snS+n42PQi75f/M/qzcmdKSDs1hrpePvd7ADPX7/e5zf4A/w+pGdl5fgf57UpxlRJsPXDU7zbBeDvAqMbhooHAyxnt6gHwuWMQMxw9uMg+h0vtpdfWe7tXljg92xFw/KJDx1xlqTd89E+BdRN+Xse4b/w3Tf0nMZnVVpHR0u2H/X65ezw1k95Pz2TB1kNc8MZ8z3KH0+QJWgC7DuctKst2OFlg5T6cxpCelbfn5PUfLuH6D5d43h8NEPQKszTfddpzJJ0t+3N/jO4b3dYDrgmE3G22vQeu27jXFTSNMbwxawtJ+TruuY/hfR3y+2HFrqA6/C3dnkyWV9n1PV+uoNfTM0t8k52+di/Hs0q/d+r8rQd56ud1nDdpHh/MSyz14+fnKy7N33IwYBGeMYbVVo5v75EMRvlo/VbSOHPdB/9wwZpjr9IAACAASURBVBvzyXE4affIr1zz/uISHjE4y3YcJn7cNPYcKZ3i6OLQQOClf5u6ntd3ZI/hoKnO89Hv0M+2JmxpmrPZdzvxA0f9P9kcSc/OU/kc6OZ2LMvB5v1peZblL/P39UT5xl+5lXBOAx0em077R6b7vdld/V7hzRy37D/qc/+7vOpIAE559k/Oesl/Ft/XjWboy7NJz3Iwf+shnp++kd1H8vbq9b4JnfzkDM/rlONZnjTd+cUKzvdqHuzLsh2HuejNBTw9LXfQvJ+tIrXSKlpeur3ofR8cTlNgPoschxNjDFe8s4h3raC5LMAT8ZpdR4o0J8aMdfsY9vLsAt+nB/PV5RzLzOGKdxf5fOhxe/SHNZz7+lz+3LCPybP/ZVGQ/WEA2j38a56xwlYnHeG56QX76ay0Gj2kZzvIynH6bXU1Y91+4sdNK7Ti2+E0xI+bxuTZgSusP124HcBv67yyENJAICLDRGSjiGwRkXE+1l8rIgdEZIX1d2Mo01MUmcRwduZEAD6PeYb2Uvw+AcUVaKjj1UlH+MZPJeB/vlpZpPM89sPaPO/T87W+2ZeaUeCJ3uHMfeKdtTE3G56/6GfrgaPsT8tg2Y7ALYvmbznIWS/9zZD/BT/E948rdzN/60Hy3/pT/DxZjpmyrMCooe4907w+X/Kx3B94twkz8uQsDh0L/OO/0Aq6P6/aw18bXNfFFqAupdWD07jnyxUFlgdy0ZsLCvRwTz6WRe+nZ7Jud2qBwL07JYM7Pl+WZz6LrBwnbR7+lYn5bog/r9rDM7+sL9AkODUjm3Nem8udX6xgy/60oJoM3/vVSjbsTSM1X4OF7YeO8+j3uQ9XOVal9qZ9aVz9/mLa5WsSunR7Mp8udP3+th0sev1ClsPJ1V5P9+e+Ppc3Z231FDlu3pdGZo7Dk6O438+EU+5i2oPWQ1hhnTPdOeT/zdgccLuwNxEkhIFAROzAJOBsoCNwuYh09LHpVGNMN+vv3VClpzgOUJOzM58F4LfYcayPvZbalF6rjsIEGuo4q4ht94sif2B47Ie17MhXJDJ/a+7Ti3dxwpJ8PXUHvfg3vZ/2PajfoBdnET9uGou3JXue8Dbv91/+6jTkuQGOnbKcK95ZlKd/wcJ/D/m9Nv6euApr/vr9il1s3pfmc93O5OPsT80g2+Fk3De5N5BDx7K47sN/mPjrBk9OwFcgcBr4dtmuAsuLavamA+xPy2T4q3MKrFu960iBuqIM6yblq7x68ux/+XnVbm77bCn/s5oKu+usZqzbx1kvzeaeL1cy8dcNQbUYy3Y6WZWUgvG6431iPQUbY/IMpzJ704E8xWrZDic/rcxtpOC9rqQMrgA6+H+zefCb1Z7P4q9eLX+AHf3JUp6bvoE1VuXut8uS+LeI9QfXfbA4T2MQt4P5cvy7UtK5bPKCIrUCLIqokBzVpTewxRjXDPEi8gVwHlB+B5r3Yb1pwa1Zd/JA1BfE2/axLO4W/pd9ER84hpJK1XAnL2jePXnTsxxUirGX6Hj+6jd+W7s3T3PXQNxl+sH2dvbHuyL2ssn+2+H7KuI6nu1gVIB9ANbsSmWwj5yKr3Gn8vNux/7wd2t44ZKuhe4DrmKFjGwHRzNzWLEzhaGdGvrd9sp3F3JJj2ae974aGLgt3pbMsawcerSoFfD8OQ7DL6v3AnsZ3qURtSpH51n/00rX2Fi94msxqEMD/vPlSqpXiuLxczux9cBR1uw64ilSm/jLBr5dvov61WILnOezRTt45Hv/Ra///W0jH85P9Lx/bvoGYuy+n1+LWgez50g67oztP8UobgNX67w3Z20lceII7vlyJTFRNjY9dTZ/btjH8h25RU3+/LXRd/HT9R/+w493nOp5//qfW1j4bzI/r9rNlX1aFCutgYQyEDQBvMsJkoA+Pra7SEROBzYBdxtjCjQrEZHRwGiA5s2bhyCpgf3q7MOvWX041zaf12Je5+7ob7g7+hvuyx7NL44+HKNSmaepqLx78t7x+TLeu7ZXSM5zLMvh8wmnvLruA//l0qXt66VJfgNB/lZFrR/K28P97/sG+D3uvC2H6Nuyjuf9tkPH/G7rDroPDT+psOR6DH15NosfGuRzXVpGDhnZDk8xZf/WdbnpkyV5Wgi5vw+H85Wpb9l/tEBDgPx8NeH1leNLOZ6VZxbBYJz63F88eo6rkCJQ8CyKrBwnXy7Z6bd4KVi7U8q24jjclcU/AfHGmARgBvCRr42MMZONMT2NMT3r1asX0gRNuuJkzu/W2Oe6n5z9iM/4jP9mXwrAf6MnszbuBl6Ofp12UnbzF5fUHxv2sz8Ch0DOCdNIo94ufMN3ZfPYKcsD7ldYa6EXZ+R2NLwviDqiL5eUTiezu6auoIdX5fqNHy8JupnoSzM2+r39zli3L6gcl1u3CTOKlbN0z4Tnr4ltutd1D/T92eV14y5uEAhmetZQNcENZSDYBTTzet/UWuZhjDlkjHE/Cr0L9AhheoIyIqERL1/Wnbhof5dGmOQ4n64Zk1nubAPA+fb5/B77AF/EPEkTKsb4Lr2fKdlkPBVRaZYvF5e/SvOklNw6mC8WF2yYkL/5biApQZQjF/b8m/9+kxng2h0Lskmru6ez2y+r97Ldq+4pNSO3wv6mj5cQrNIYgdXf8CQdHpvOr9bQ0c8FGBH4wyJ2MluwtWB9la9e5cYYvlyyM+Tf3VAWDf0DtBWRlrgCwGXAFd4biEgjY4y7JmgkEJo+/cXw9S39+HhBIlF2m89RRI9QlQuyJgBwsmziqegP6Gtbz7y4O0k1lfjJ0Y/Hc64hJ6SXWJ0odibn3ujHfVuwj8iNRbgxBnPTCFQpDwXL29+YFZoxe9JKYZC90hiBdV+q/+bYt362jFsHtA7YF+adOUULBJe/s5COjaoH3CYzx8EH8xLzDJMRqjxtyO5SxpgcEbkD+A2wA+8bY9aKyARgiTHmR2CsiIwEcoBk4NpQpaeoOjepwfMXu8pzCxtOeplpx4isp7naPoMzbSsYYF/JlVF/cGXUH3yQM5Q1zpb85uwJQB1JZbtpQPkdCECpgnMlHC6k2WxxlVbZfKi9GYJAmL/TpreDR7MY+MLfeYqcQkkq2jy5PXv2NEuWBP90VBqKUlbpYjjfNo/noicTK/6fItY6WzDN0YdfnX3YZhqVLJFKlaLKMfaQ9GLOr2XdKmw76L9yW+X15Pmduapv8VoNichSY0xPX+vCXVlcIfxvVHBN/nIJ3ztPpX3mx3TNmMxz2ZeRYqqw0tmKf5ztPFt1sm3n/ugv+Sv2P9xp/waAKHJKfbA7pYqqLIIAoEGgnNAcQZCKnisIrI0k0UQOMdT2D5fY/yZacn94qaYSW0wTDMIyZ1uOE8d+U5OfHX05UoH6LiilSteT53XiqlPii7VvoByBBoIgrdudmqfXZtNalUgqQkuOQGLI5s6ob7g96kdSTSWqS3DHnePozBeOgdSXw9SXFKY7erHSuFoy1ecwGcSQSpVSSaNSKvw0EFjCFQjcLn1rAYsTk7n59Fa8HdLx6g0NSaapHKCapNNJErk3+qsiHSHb2PncMZDjxFGHVFaYNsx2diHJ1KM6x2kiB1lvmnOS7CTRNCCDvD0/BSdGSw+VKjc0EFjCHQiSj2Uxf+tB9h7J8EzkEi4tZC8jbIvYY2qTSTRvxLzqWTfd0YsYshloL9qAZu5920oSrW17OGriiCaHVCqz1TShFmk8lH0DS0370vwoSqkgtKxbhb/uHVCsfTUQhMA7s//l6RBNZVgyBq8pOngg6gvqy2G+d5zKWbalXGH/ky2mCWtNPIKTdpLkGSKjry34z3PQVKeuuJq/fe/ox3bTgFXOVqxzxtPbtp7aksYqZys62RJZ52xBHUnlb2dXbBjuifqKY1Tizqhv2eRswgbTnPqSwvPZo1hm2hVyZqUiV7XYKFaPH1qsfQMFAu3tdMKRPK+fy7nc826OM4HHc64LuPdI2zyG2JfyZs5INphmdJMt7KcmO00DADpJIuOiPide9nlOdb7d/3wHhWln20U7q8P5t7FPeJb/6ehGR9t2GoprLJrdpjYOYydWsqkvKTiN0CHzAzKJAcCGk1qkkUU0bSWJxnKIGnKMeNlLvOzjOLG8lHMxlciip20jG53N+Me0x9WS3Ul9UthH7WJ/DqUqMg0ExWTKwyDiIfCjsz8/Ovt73ucvAlpr4rkq+yHP+yvsf9BZttFYDnHA1KCDbQfJphrznJ3JJJoc7Fxm/4soHESTQ2M5xFTHANabFixwdiTDxNDLtpGlzrZ8F/s4jcQ1CmT+Iq3dpi4tZB/1xDXkr00MG+OuLdJnOy+IgLXL1OGwqcYaZzwAK0wbanKUunKEVrLHk650E0MKVWkkycxxdGaKYyBrTTx9bOvpJltZb5rzZPSHAKSYKvzg6MdMZw/STGVysPGvacxx4ogih1Nta6jBUQ5Rg7nOzlZKfHW0MsSSTSUySaFakT67t1Nsa6lEJsucbUt0HBUGIep/p0VDxTR59lae+WUDN57akuZ1KjOgXX1O/69rWsv61WJJzcgu0DtTFa657KMGx+hi28b3jv4cJ87nduOjPuCaqBkFlh8zsaRShWRTjQ8dQ9nqbMwK04b/s89gQvRHbHU24iA1qMsRWttyx7lPNlWpLSWbi7aoHEawi//f3zETy3HiqCdHWOFsTTdb3t6tK5ytSTbVaCV7WOjswDEqEYNryIaB9uWMy76Ji+yzGWhbzgJnJ9KozMX2vMNpL3W2pYfNNXHKoMz/stU0QXAiuFqexUgOQ2xLGGxfSnWOcZRK9LJtYrUznkTTkCmOgew3NbHjZKNpTgzZZBGFYKht5dDSqFzotTjPNpfx0R9RU1z9Co6aOKpKBudnTmCF1RLOP1eAbCO7Odu+iH2mFntNbTaZpmw3ruG7q3OUOpLm1XHTuwg1sLNti7gr6hu+dZzGTOfJbDVNgtovFEJVNKSBoJjcgeCm01ry8AjXULb70zKIttmoVSWGzxZt5+HvXOOs16oczciujflowfZwJlkVIoZs6pHCHupQhyNcZv+L2pLGVtOYWc6upJiqZBJDNlG4byQNSGZ01DTqyhF2mPocMtVpIIeZ4ejBGtOSLKIBw9SYJzlkqlOJTDKI4YCpSXU5xgjbItab5vzp7E5H2c4Qe+5w4WmmEg5snptjafoiZwAtbXvpKRsDBqPS4jSCTQy3Z43lGLHMdXYhCgevRE9iqL3w3/MWZ2NayR72UouJ2ZfTz7aWy6JmFSstm5xNaGdzFUceMDX4zdGTrxxncJRK7DW1OdW2mleiJzHH2YXB9mV+j/NS9sX84OznCTZuMWTTWA6SaBpiw9BJEqkvhzloapBoGnJ71A9cap/FX85uTHf0Zp1pzhFTBRuGGnKMKBx+g03V2CjWaCAoP4Hg7b+38uyveQOBt08XbvdMuPHq5d1pXruy3/luFz44iL7PRt5ooKo4cp9km3CAnraN7DV1OEh1HNg4airTxraL9rKTJnKQHaY+G53N2ENtmspBFjrzfldrcBQbTtrbkvg0+hmixEm6iaGSZPGzoy+n2Vaxy9RjZNaTtJB9HDQ1EAwtZB97TB0eif6UkfYF7DW1PPU53pJMXfaY2vSybSqwLr/+Ga+QTDXSrVxgJ9nGzVE/M9K+gAwTTZwEHqBup7MezWy+R/9d5DyJPjbX4G2JzgZUkkwaSOCZ6dzmOzryqeMskqnOLfafOM22KuTBs1vG2z6L7UIVCLSOoJhOskYO7NK0ps/13l+TEV0aYRO4rn88Ww8c440rT6bz47nzxzasEcfCBwcxe9MB7remOyyrsV5URZNbnLGLeuxyFpyf46CzBgsp+HCSZOoXWObuqb7Q2ZE2mZ8GPLP3U2qKcd2kxmaPYWz2mKBSXoOjXGv/jeuipmPDkEYl3s0ZzgeOs31uv9a0zHP8mqQRSzbn2BdQhUymO3uxy9Qt4cRQhlNtaxhoW04nWyLVSCeNSvzt6MoG04wlzvZ5OmW6A2kdjvBA1BdcbJ+NLV9QWOxsT2/bxjzLFjg6cop9HYnOBvzl7ManjrNIMVXpa1vPpJhX2eRsggM7+03NgHU3vqY7LQ2aIyiB7YeO0aKO756787Yc5Mp3F/H0BZ19Ti3nHrLih9v707VZbjAZ980qvvhnJ8sfHUz3J2fQuEYcz12cQJXYKM/E6N5sAuVgvhWlIpoNJ84y6HxZKdrO+ieHFWtfzRGEiL8gANC/TV1+u+t02jUIPDaQdxAAmHhRAhMvSgDgqfM7c0a7ejSrXdnvxBl3DGzLq39sLmLKlVKlqSyCQChpIAih9g39N81rUD024GQYAP/nNdxsXLSdVU8MoUpMFMYYXp65mdf/2kKnxoEnt3Dr3rymZzLtvq1qs/Df4k3WrZQ68VTsMFaB/X73Gcx94Mwi7VM9Lhq7TYiy27hncDu+GN2XoZ0akjhxBPcNDTzkw/8u7eZ57WsykOa1C2/ip5QKryqx9pAcVwNBmNSoFE3TWsW/+dpsQt9WdTzvbz+zDS9cUtR5E1y+GN2X2fcXLSgppcpe7SoxITmuBoITSOOaBTtfnWQVT9WpmvsF8i6yeuycjp6A8vOYUws9x6onhuR5H20XLu3Z1PM+Jsr3V2r+uIEFlt1wastCz1cUb1x5cqkeT6lIoYHgBNKvdV2+ubUfPVrU8ix76vzOJE4cQbW4aM+yh4Z3YNIVrptm12Y1PMs7N8l9nThxBMsfHcw3t57Ci5d0pW39qmx7djjV46L5/MY+jOrZzLOtTVxFTRee3IRZ9w7gzkFt86Tri9F9aVwzbxO/a05pwaPnFGziGIxXLuvmc7l3DikY53Vr7HfdLWe0JnHiCPq1Ltoxy6NzEnQa1BOF+7dW2rSy+ATTo0UtJl1xMlMW72Bkt8a0rpfbaunv+wZwJD2bmCgbIxIa0a/1YGoFyGrWqhJDjyq16dGiNhf1yH3q79emLqe0rsOKnSmMGdSGOZsOAtArvjaNa1bi7sHtWPjvITbsTePJ8zt7btD/GdyOF2cU3rHI2/xxA/lk4XbGDGzD5n1HSWhag8wc19AdMVE2snJyh/Hwlxvx59kLu/DDit0+13Vo5Mo1fXhdb9o98muedT+POZXnpm9gzmbX5x4/shOP/7i2SOcuK9PGnkqnxjXIyF7CzPX7wp0cVU5pjuAE1LBGHHcPbpcnCICruWuCVwc4X0GgcQ3fY/vkJyL8dvfpnJPQ2DMAn/ezytSbT2Hl40MY2TX3qXvMoLb0beUa4dPdIurus1zDTt8/LLey2ztH0bhmJR4YdhKVY6Lo2qwm4v1EZOD723MHyPM+//JHB/tN+38vTmDlY0OoHJP3OWjN+KG0rFvF8/kgb3A5q0MDxg5qS6fG1RneJfcpu1Pj6nx+Y588x3IfB6B3y+KPahoo11IYV1pdubzY6OB/6s9d1IWPr+9d7PMq34Z0bBBw/dh8OWlfJEQ5Ag0EKo/pd5/uszw/EPdNsWd8rUK2hC9Gn0LixBG0beB64r5jYBs+uaE3tw3IHVjs7sHtWPTQIBY+OMjnMWKjbJyT0IgPr+tFt2Y1qRRdsCVF/iDXpr4rKD5zQRcu6dmMGpVdRWU3n97Ks03V2CjuOsv1Y+zYqGDT30dGdOCewe0Qya2oP7tzQ3q0qEW/NnU922166mwGe/3o7SI8NPwkz/v7hrYPeFNInDjC89omUiCQtK0fuG/KoJPqc+OpLbnljNzPdr+PVmUXdG/iqfPxbnXWqXENTm+X22P5pADNoENl41PDAtb5zLj79GId9/1rc/tT1a1a8opX7yLSQGpUiqZB9bwPWa9f0T3P+3sG+5+Lo5vV38gWotFHNRCoPKrHRRcozy/MgPb1SZw4gjb1i37DsNuE09q6bjqz7zuTj6wn0QbV42joJ3ciIrx+xcmem+/lvZsDBYuG7F6/Gn896B8c3oHvb+/PN7eeAsB53Zqw4clhPj9LvNdTfsu6VUicOII3/69Hgae0/Ol44dKujD69tef97We2YfLVPT25IV/c9SA942vx5c2n0Mo697SxpzLjnjMKbP/5Tbk5kklXnswj53TMk+NpUaeKp17ILS7aRvW4aBInjuD2MwuO8Hl9f1dlfqWY4jdZvLJPc7/r7h1S8PN3bVaTS3s2JcZuK9DZ0pv7QQJg5eND2Pbs8ELTkjhxBANPasCE8zoBrqJMgN7xtVkzfmjAPjn3DG7HTK/r/uT5ruHCrzql4KgB3v78j2ufa/rFE2XP+z2Jttvo38Z/HdTKx3IbZrgfZEKUIdA6AlV+NK9TmeZ1it6k9pERHbh/WHui7TYmXtjFcyP+/e7TefH3jfyyei/tGlRj6wHfo3h2y3fDifORwwhGy7pVSDp8PM+yB88+iSZ+AuudZ7WlX5s6LEk8TO+WtbnozdwhRM7r1oSEpjWJt67HmEFtuHvqSlrVrer5zO6pUseP7ES/1nVZ+fgQbOI//SMSGnH7567XbetX5cHhHXxuVyXWdVuIs4qTTm9bj7W7Usly5B1WvX+bOnx2Y1/PcCn5fX5TH/q1rsuA9vX5YcUuUjNyeHh4BxpUj2Vncjpdmtbg0p7N6P1M7oCLwzs35OYzXEGzYfWCDwJ//OcMTy/7P/9zBjFRNmpUii6wXSDDOjXksR/Wck2/eJ67OIHYKBuxUXamjT2NCT+t4/152wDY8OQwomzCzPX7OKtDA6LsNtaMH4pY1+iyXs2ItttInDjC7zVoVa8qs+4dQLPalTmamcMH8xIBqBYXxeAODRjaqSHx46bRxauhBrjqoWpUjmbrM8P5bvkuWterwtdLk3z2ASoNGghUhWezCXE2183vst65T6Ct61XljSt7APDQd6uB0D1RAfxxzxmewQav6tuCPzfs54Lugceu7xVf2/Nkennv5szelDt6pnc9wwXdm3JB99wK+xtObcklPZvx9t9bPTmiYG6IH1zbi+s+/IcPr+9N9bi8268dP5TFicme844Z2JYom3D7mW24e3C7PDe7d6/uyaltXTmy/JX2bv1au9YP7tggT1EZQM3KrmKZ+vlu9ud1y71e3jm6oZ0a4HCaPPVereoFLiL7v77N+XThDh4aflKeHFn96nF5it+8PXZuR08gcAfUYZ1z64OqxubeMqPtuTm/NeOHYoxhZ3I6w1+dk+eY7pxkjUrRrHx8CBv3ptErvpYnJ+mdlik39aVJzUqeByK7Tbi4R1OSj2UBcFnv4IqiikoDgYoI9w1pjzEUemP25e6z2tGnVeEVvjavG1ez2pXzFCW4BSoKePbCLkGnSUSoUSma+4edVPjGXs48qb7fm2CV2CjObJ87QmmlGDv3DClYt5B//7kPnMnulAzOnzSP+tVi2Z8WeOiU/Cac14nHfnC1uspfHHha27rM2XyQt6/yOVZaQONHdqZDo+pBl+OXhDtAdGwcTc8WtViyveCQ3OAKBoEaD5zip7ly7Soxfv/fSoOOPqpUGUnPchBllzxPkhVJm4d+oU39qky/y3dFbdLh41SLi2bEq3PYnZLOv88Gf+Na+O8h6lWLLdDSLSPbQWp6doGcgy+LtyXzyPer2bTvKLPuHZCnTqco3Dmf4t54cxxOznltrifXVl7oxDRKqRJz3ysKa8KYY9UlRFXQgFfSQFBe6TDUSqkSC7YNe0UNAG7/vTgh4BDzJyINBEop5aU8FeeUlYodupVSSpWYBgKllIpwGgiUUirCaSBQSqkIp4FAKaUinAYCpZSKcBoIlFIqwmkgUEqpCFfhhpgQkQPA9mLuXhc4WIrJqaj0Orjodcil18LlRL4OLYwx9XytqHCBoCREZIm/sTYiiV4HF70OufRauETqddCiIaWUinAaCJRSKsJFWiCYHO4ElBN6HVz0OuTSa+ESkdchouoIlFJKFRRpOQKllFL5aCBQSqkIFzGBQESGichGEdkiIuPCnZ7SJiLNROQvEVknImtF5E5reW0RmSEim61/a1nLRUReta7HKhE52etY11jbbxaRa8L1mYpLROwislxEfrbetxSRRdZnnSoiMdbyWOv9Fmt9vNcxHrSWbxSRoeH5JCUjIjVF5GsR2SAi60XklAj9Ptxt/SbWiMgUEYmL1O+EX8aYE/4PsANbgVZADLAS6BjudJXyZ2wEnGy9rgZsAjoCzwPjrOXjgOes18OBXwEB+gKLrOW1gX+tf2tZr2uF+/MV8VrcA3wO/Gy9/xK4zHr9FnCr9fo24C3r9WXAVOt1R+s7Egu0tL479nB/rmJch4+AG63XMUDNSPs+AE2AbUAlr+/CtZH6nfD3Fyk5gt7AFmPMv8aYLOAL4Lwwp6lUGWP2GGOWWa/TgPW4fgTn4bohYP17vvX6POBj47IQqCkijYChwAxjTLIx5jAwAxhWhh+lRESkKTACeNd6L8BA4Gtrk/zXwH1tvgYGWdufB3xhjMk0xmwDtuD6DlUYIlIDOB14D8AYk2WMSSHCvg+WKKCSiEQBlYE9ROB3IpBICQRNgJ1e75OsZSckKzvbHVgENDDG7LFW7QUaWK/9XZOKfq1eBu4HnNb7OkCKMSbHeu/9eTyf1Vp/xNq+ol8DcD21HgA+sIrJ3hWRKkTY98EYswt4AdiBKwAcAZYSmd8JvyIlEEQMEakKfAPcZYxJ9V5nXHncE7a9sIicA+w3xiwNd1rKgSjgZOBNY0x34BiuoiCPE/37AGDVgZyHKzA2BqpQ8XI0IRcpgWAX0MzrfVNr2QlFRKJxBYHPjDHfWov3WVl8rH/3W8v9XZOKfK36AyNFJBFX8d9A4BVcxRxR1jben8fzWa31NYBDVOxr4JYEJBljFlnvv8YVGCLp+wBwFrDNGHPAGJMNfIvrexKJ3wm/IiUQ/AO0tVoKxOCqBPoxzGkqVVY55nvAemPMS16rfgTcLT2uAX7wWn611VqkL3DEKjL4DRgiIrWsp6kh1rJyzxjzoDGmqTEmHtf/8Z/GmCuBv4CLrc3yXwP3tbnY2t5Yyy+zWpC0wXUZLwAAAtRJREFUBNoCi8voY5QKY8xeYKeItLcWDQLWEUHfB8sOoK+IVLZ+I+7rEHHfiYDCXVtdVn+4WkVswlXb/3C40xOCz3cqrmz+KmCF9TccV/nmH8BmYCZQ29pegEnW9VgN9PQ61vW4KsO2ANeF+7MV83oMILfVUCtcP9otwFdArLU8znq/xVrfymv/h61rsxE4O9yfp5jXoBuwxPpOfI+r1U/EfR+A8cAGYA3wCa6WPxH5nfD3p0NMKKVUhIuUoiGllFJ+aCBQSqkIp4FAKaUinAYCpZSKcBoIlFIqwmkgUBFLRI5a/8aLyBWlfOyH8r2fX5rHV6o0aSBQCuKBIgUCr16p/uQJBMaYfkVMk1JlRgOBUjAROE1EVlhj19tF5L8i8o81Nv/NACIyQETmiMiPuHqnIiLfi8hSa7z70dayibhGu1whIp9Zy9y5D7GOvUZEVovIKK9jz5Lc+QM+s3rCKhVyhT3VKBUJxgH3GmPOAbBu6EeMMb1EJBaYJyK/W9ueDHQ2rqGIAa43xiSLSCXgHxH5xhgzTkTuMMZ083GuC3H1+O0K1LX2mW2t6w50AnYD83CNiTO39D+uUnlpjkCpgobgGndnBa6hvOvgGlsGYLFXEAAYKyIrgYW4BiVrS2CnAlOMMQ5jzD7gb6CX17GTjDFOXEOExJfKp1GqEJojUKogAcYYY/IMriYiA3AN5+z9/izgFGPMcRGZhWusmuLK9HrtQH+fqoxojkApSMM1vafbb8Ct1rDeiEg7a1KX/GoAh60gcBKuKR7dst375zMHGGXVQ9TDNYvYiTOKpaqQ9IlDKdfonA6riOdDXHMYxAPLrArbA+ROZehtOnCLiKzHNSLlQq91k4FVIrLMuIbCdvsOOAXX/LcGuN8Ys9cKJEqFhY4+qpRSEU6LhpRSKsJpIFBKqQingUAppSKcBgKllIpwGgiUUirCaSBQSqkIp4FAKaUi3P8DUOAWr5H+tWcAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_G-ii1Ktgzc3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        },
        "outputId": "7cdd08ce-6a54-491a-a8cf-d0d528817a30"
      },
      "source": [
        "plt.plot(np.arange(1, NUM_EPOCHS+1), train_acc_list, label='Training')\n",
        "plt.plot(np.arange(1, NUM_EPOCHS+1), valid_acc_list, label='Validation')\n",
        "\n",
        "plt.xlabel('Epoch')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.legend()\n",
        "plt.show()"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEGCAYAAABiq/5QAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU9b3/8dcnC9nJHrYQwr7KIhERFEGtC1pQigou1WpFvfbWpd7Wtl6ltvZXl9vaWy0VS9W6oVal4nWnClZQWQwQFtkMEMISAtnXST6/P84AAZIwkJyZJPN5Ph55ZObMWT4Zhne++Z7v+R5RVYwxxgSPkEAXYIwxxr8s+I0xJshY8BtjTJCx4DfGmCBjwW+MMUEmLNAF+CIlJUUzMzMDXYYxxrQrK1eu3K+qqccubxfBn5mZyYoVKwJdhjHGtCsisr2x5dbVY4wxQcaC3xhjgowFvzHGBBkLfmOMCTIW/MYYE2Qs+I0xJshY8BtjTJCx4DfGmLaoeBe8dx9UFrX6rtvFBVzGGBM0SvfAZ7+Hlc+CKvSZCAMvbtVDWPAbY0xbULYPPv8jLP8r1NXCqGvhnHshsVerH8qC3xhjAqm8EJb+Eb56BjxVMGImTLgXkvq4dkgLfmOMCYSKA7DsSfjyaagph9OuhHN/Bin9XD+0Bb8xxvhTZRF8MQe++DNUl8DQaTDxPkgd6LcSXA1+EbkTuAUQ4BlVfaLBaz8BHgdSVXW/m3UYY0zAVZU4rftlf4KqYhj8XZj4c+gy1O+luBb8IjIMJ/THADXA+yLyjqpuEZGewIXADreOb4wxbUJ1GXw1F5b+L1QehIGTnRZ+txEBK8nNFv9g4EtVrQAQkcXANOBR4A/AT4F/unh8Y4wJjJpyKPgGvl0MS5+Eiv3Q/0In8HuMDnR1rgZ/DvCwiCQDlcBkYIWITAV2qepqEWlyYxGZBcwCyMjIcLFMY4w5RbVVsH8TFGyEfeth30Yo2AAHtwPqrNNnEkz6BfQcE9BSGxJVdW/nIjcD/wGUA+uAUGAEcKGqFotILpB1oj7+rKwstTtwGWMCxlMDB7YeHe77NsCBbaD1zjohYZDcH9IGQepgSBvs9N8n9w1Y2SKyUlWzjl3u6sldVZ0HzPMW8FtgL3A5cKi1nw6sEpExqrrHzVpMO7TtU8h+BbQucDXEpDpD7KISAldDe1JX64ThPm8wHswN7L9fS3mqYP8WKNwM9R5nmYQ4Y+zTBjsjctK8IZ/UF8I6BbZeH7k9qidNVfeJSAZO//5YVf1jg9dz8aHFb4JMdRl8/KBzBWN0MkTGB66Woh2wZRFc8yok9Q5cHW1NfZ0T6vs2HGn97tvodHvU13pXEohPh9DwQFbaMiFhkNwPBl5yJOCT+0N4ZKAraxG3x/G/4e3jrwXuUNXWn23IdCzbl8GC251QOetHcN79EB4VuHq+/QxevQ7+ej7MeBkyxgaulkCor4finccE/AYn4D1VR9ZLyHC6N/p/50hApgwI7L+daZKrffytxfr4g0BtJfzrN7DsKWduksvnQK9xga7KsX8LvHyVE4BTn4LhVwW6oubVVMDmDyDnTdj+udM6P1WeavBUHnneuQekDjoS7qmDnQuPImJbXrdpdQHp4zfGJ7tWwlu3Oa3IrJvhOw+1rSBJ6Qc//BhevR7evAX2b3ZGaTQzKs3vaqtgy0dO2G96H2orICYNBlwCnWJOfb+h4ZDS/0jA27mODsGC3wSOpwYWPwL//gPEdYXr34K+5wW6qsZFJzn1vXM3LHkUCrfA5X8ObFeGpwa2/gvWvQkb34WaUuecyPCrYdg06DUeQkIDV59psyz4TWDsWQtv3Q5718LI6+Di3wb2JK4vwjrB1CedvwA+nu10/cx4GWLT/FdDXS1sW+wN+3ecS/8jE2DoVGeESe9zIdT+W5vm2SfE+FedBz7/A3z6CEQlwsz5zoiJ9kIEzr7bGbr35ix45nxnxE+XIe4ds84D2//tdONsWAiVByCiMwy61An7PhPbzTBC0zZY8Bv/KfjG6cvPXwXDvgeTH3e6UNqjIVMgoSe8PAPmXQhXPgf9L2i9/dfXwY5l3rB/G8oLoFOs80ty6DTodz6ERbTe8UxQseA37quvc6ahXfSQc6Jx+rNOH3R7130U3PIveOVqePlKuPgROHPWqe+vvh7yvoJ1b8G6BVC2B8KiYMBFzvvV/0IbHmlahQW/cdeBbbDgP5zW68BL4btP+LdP3G3xPeAH7zujfd77L+cKz4v+n+/97Kqwa5XTZ79uAZTkQWiEMx5+2DQYcHHLRuUY0wgLfnPEsZfbF2yA4l0t2+e+9RASDlc87Yw2aUtDIFtLRCxc/SJ89IBzR6UD38L0v0Fk58bXV4Xdq71h/5ZzdXBIuNN9c/4DTndOU9sa0wos+IORr5fbJ2Y6F1NJC4YEDrncufo2vkcrFN6GhYTCRQ87Y97/7yfwt4uck74J3pllVWHvOm83zpvOL9iQMOfE7Ln3OSdqbYy88RML/rasvh6qi1u2j6qSo6eM3bf++Mvt4zOcqzD7X3BkVsGUAdApumXHDkajb3R+Yb76fXjmPLj0997Af9N53yUEek+A8XfC4Cnt9+S2adcs+NsCVSjJP34+lIJvoLa89Y4T192ZMrb3hCOX3acOhIi41juGcVrxP/zYOeH72vWAOBdTnXkrDJ4KsakBLtAEOwt+f1KFsn1Hh/u+DU6LvLrkyHoxaU4on36901UgIad+zPAoJ+RTB1lXgj+lDoBbPnGmls44Czp3C3RFxhxmwe+migPOn/gNQ77ywJHXo5IgbYgz6VfqIOdx2mD787+jiE7qGMNWTYdjwe+W6lJ4/ruwNwci4p1AHzLlSB962mDnJh8dcZSLMaZNs+B3Q30dvPFDp4V/zWvOhTcW8MaYNsKC3w0fPeBMjTv5ceeqS2OMaUNacNbQNGrlc85FPGNuhTG3BLoaY4w5jgV/a9q22Ll4p98FcNFvA12NMcY0ytXgF5E7RSRHRNaJyF3eZb8WkTUiki0iH4pIdzdr8Jv9W5wx28n9ncv1bU50Y0wb5Vrwi8gw4BZgDDACuExE+gGPqepwVR0JvAM84FYNflNxwLkna0gYXDO/7d9QxBgT1Nxs8Q8GvlTVClX1AIuBaara4EolYoC2f7f35nhq4LXvH7kbU2JmoCsyxphmuRn8OcA5IpIsItHAZKAngIg8LCI7gWtpzy1+Vfi/eyD3M5jyJGSMDXRFxhhzQq4Fv6puAB4BPgTeB7KBOu9rv1TVnsBLwI8a215EZonIChFZUVBQ4FaZLbPsSfj6BTjnXhhxdaCrMcYYn7h6cldV56nqaFWdABwENh2zykvA95rYdq6qZqlqVmpqG5zU6pv34MP/hiFTYdIvA12NMcb4zO1RPWne7xnANOBlEenfYJWpwEY3a3DFnrXwj5uh+0i4/C8QYqNijTHth9tjDt8QkWSgFrhDVYtEZJ6IDATqge3AbS7X0LpK9zg32I6Mh5nzbc56Y0y742rwq+o5jSxrtGunXaithPnXODNs3vQ+xHUNdEXGGHPS7CojX9XXw4LbnRtjX/0idBsR6IqMMeaUWPD7avHvnPulXvArGHxZoKsxxphTZmclfbHmdVj8CIy8zrlXqjHGtGMW/Cey8yv45x3OPVMv+4PNq2+Mafcs+JuzZ61zMrdzd6dfP6xToCsyxpgWsz7+YxVude6Tu26Bc9vEyHjnLlp2H1xjTAdhwQ9wcLtz4nbdm7B7tbOs55lw8e9g6DSI6xLY+owxphUFb/AX5zmt+nVvwq6VzrLup8OFv4Ehl0NCz8DWZ4wxLgmu4C/dA+v/CTlvws4vnGVdh8MFs2HoFTalsjEmKHT84C8rgA3/hJy3YPvngELaUDjvfhhyBaT0C3SFxhjjVx07+D96AJb+CbQeUgbAuT+DYdMgdWCgKzPGmIDp2MHf/XQ4+x4n7NOG2Bh8Y4yhowf/0MudL2OMMYfZBVzGGBNkLPiNMSbIWPAbY0yQseA3xpggY8FvjDFBxoLfGGOCjKvBLyJ3ikiOiKwTkbu8yx4TkY0iskZE3hKRBDdrMMYYczTXgl9EhgG3AGOAEcBlItIP+AgYpqrDgU3Az92qwRhjzPHcbPEPBr5U1QpV9QCLgWmq+qH3OcAXQLqLNRhjjDmGm8GfA5wjIskiEg1MBo6d6/gm4L3GNhaRWSKyQkRWFBQUuFimMcYEF9eCX1U3AI8AHwLvA9lA3aHXReSXgAd4qYnt56pqlqpmpaamulWmMcYEHVdP7qrqPFUdraoTgIM4ffqIyI3AZcC1qqpu1mCMMeZork7SJiJpqrpPRDKAacBYEbkY+ClwrqpWuHl8Y4wxx3N7ds43RCQZqAXuUNUiEXkSiAA+Emea5C9U9TaX6zDGGOPlavCr6jmNLLNbXhljTADZlbvGGBNkLPiNMSbIWPAbY0yQseA3xpggY8FvjDFBxoLfGGOCjAW/McYEGQt+Y4wJMhb8xhgTZCz4jTEmyJww+EXkuyJivyCMMaaD8CXQrwY2i8ijIjLI7YKMMca464TBr6rXAaOArcBzIrLMe3esONerM8YY0+p86sJR1RLgH8B8oBtwBbBKRP7TxdqMMca4wJc+/iki8hbwKRAOjFHVS4ARwE/cLc8YY0xr82U+/u8Bf1DVJQ0XqmqFiNzsTlnGGGPc4kvwzwZ2H3oiIlFAF1XNVdVFbhVmjDHGHb708b8O1Dd4XuddZowxph3yJfjDVLXm0BPv406+7FxE7hSRHBFZJyJ3eZdd6X1eLyJZp1a2McaYU+VLV0+BiExR1bcBRGQqsP9EG4nIMOAWYAxQA7wvIu8AOcA04OlTrtoYYzogT109+UVV5BaWs/1ABTsKy7lubC96Jce06nF8Cf7bgJdE5ElAgJ3A933YbjDwpapWAIjIYmCaqj7qfX5qFRtjjMuqauvIL6pkf1kNkeEhxEWGExcZRlxkGBFhoS3e944DFWwvrGB7Ybnz/YDzeNfBSjz1enjdiLAQxvdL8X/wq+pWYKyIxHqfl/m47xzgYRFJBiqBycAKXwsTkVnALICMjAxfNzPGmGZ56urZV1pNflEl+cVV5BdVsrvh4+IqDpTXNLl9p9CQw78E4iLDiY048vjIcud5RFgI+UWVR4X73pLqo/bXOTKMXskxnNYjnsuGd6NXUgy9kqPplRxDWlwEISGt30j2pcWPiFwKDAUiD7XUVfWh5rZR1Q0i8gjwIVAOZOOcGPaJqs4F5gJkZWXpCVY3xgQ5VaWk0kNBWTX7y6opLKuhoLSK3cVV7PIGen5RJXtLqqg/JlHiIsPoHh9Ft4RIRvRMoHt8JN3io0iNi6DaU09pVS2lVR7Kqj2UHHpc5Tm8fMeBCkqrnNfKqj3oMftPi4ugV3I0Z/dLJTM5mgxvsGcmR5MQ7dMp01Z1wuAXkb8A0cAk4K/AdOArX3auqvOAed79/BbIO+VKjTFBp65eOVhRw/6yavaXer+XVbO/rOFjJ+QLy2qoqas/bh+dwkLoFh9J9/gozuqbTI+EKLp5Q955HElcZHir1ayqlNfUUVpVS2VNHV3jI4nu5FMb2298qWacqg4XkTWq+isR+R/gPV92LiJpqrpPRDJwTuiObUmxxpj2q8ZTT1FFDQcrajlYUUNRRQ0Hyo88PlhRe+T18hoOVtRQXFl7XOscIDxUSImNIDm2EymxEQzq2pmU2AhSvM9TYiNIiXMeJ8d08us5RREhNiKM2Ii2FfYN+VJZlfd7hYh0Bwpx5uvxxRvePv5a4A5VLRKRK4A/AanA/4lItqpedLKFG2PapqKKGr7YdoBlW/eTvbOIwvIaDpbXUF7TdE9vZHgIidGdSIjuRGJ0OIO7dyYxOpzE6E5HBXxKbASpsRF0jgqzASIt4EvwLxSRBOAxYBWgwDO+7FxVz2lk2VvAWydTpDGm7Sqv9rA89wDLthaydGshOfnFqEJUeCijMhLomxp7ONATYjqRdOhxdCcSY5xwjwxv2UgZc3KaDX7vDVgWqWoRTuv9HSBSVYv9Up0xps2p9tTx9Y4ilm4tZOkWp1XvqVc6hYYwKiOBu84fwLh+yYxIT6BTmN3DqS1qNvhVtV5EnsKZjx9VrQaqm9vGGNOxeOrqyckv4fMt+1m2tZDluQeo9tQTInBaegK3TOjDuL7JZPVKIqqTtdzbA1+6ehaJyPeAN1WPHaRkjGlL6uuVwvIadhdXkl9USUFpNTV1Sn294qlX6urrqauHuvp657kqdXXOa/Xq/X54XaWoooYVuQcprfYAMKhrHNee2YtxfZMZ0yeJzq04Gsb4jy/BfytwD+ARkSqcq3dVVTu7Wpkx5jilVbXkF1WR7w323Q0fF1exu6iq0SGNjQkLEUJDhLAQIcT7PTQk5PDy0BAhulMo3x3ZnXF9kxnbJ5mU2AiXf0LjD75cuWu3WDTGj0qqatm4u5T1+cVs2lfmXGHqDflDLe9DQkOErp0j6RYfyYj0BC4e5oxX7+4dn57WOYKI0FBCQ70BL0eC3gQvXy7gmtDY8mNvzGKMOTmqSn5xFevzS5yv3cWs313CzgOVh9dJiA4nPTGKXskxjOub4lyIlBBF9wTne2psBGGhdgLVnBxfunr+q8HjSJzZNlcC57lSkTEdUI2nni37yli/u0HI55dQUuW04EWgd3IMw9MTmHFGBkO6dWZI986kxUXYeHXT6nzp6vluw+ci0hN4wrWKjGmnVJWCsmp2Hqgk72AFOw9UsG1/ORt2l7JlXym1dc7YiMjwEAZ17cxlI7ozpFtnBnfrzKCuccS04Ss9TcdyKp+0PJwpl40JOsWVtew8UOEN9kp2egN+50En7Ktqjz6x2qWzM53AxIGph1vxmckxhFofuwkgX/r4/4RztS44d+waiXMFrzEd0qG+95XbD7JuVzE7DlSw44AT8Ie6Zg6JiwyjZ2I0fVNjmDgglZ5J0WQkRdMzKYr0xGi7ItW0Sb60+BvOoe8BXlHVz12qxxi/q/HUs353CSu3H2TV9oOs3H6QPSXOFFWdwkJIT4yiZ2I0ozIS6Jl4KNij6ZkYTXy0jWM37Y8vwf8PoEpV6wBEJFREog/dWcuY9uZAeY0T8DuckF+9s4hqj9NF0yMhijG9kxjdK5HRvRIZ1DXORs2YDsenK3eBC4BDd96Kwrm5yji3ijKmtdTXK1sLyljhbcmv2n6QbfvLAecCpqE94rn2zF5kZSZyekYiXeMjA1yxMe7zJfgjG95uUVXLRCTaxZqMOWV7iqtYnVfEmrwi1uQVs3pn0eF++aSYTpyekciVWT0Z3SuR4enx1gdvgpIvwV8uIqer6ioAERmNcw9dYwLqYHkNa3YVs2ZnEavzilmTV8S+UmcOwdAQYWCXOC4d3v1wt01mcrSNiTcG34L/LuB1EcnHmaenK3C1q1UZc4zyag9rdxUfbsmvyXNG2xzSJzWG8f1SGJ4ez/D0BIZ272yteWOa4MsFXMtFZBAw0LvoG1WtdbcsE8xqPPV8s6eU7LwiVu90vrYUlB2+gXWPhCiGp8czc0wGI9LjGZYeb7NEGnMSfBnHfwfwkqrmeJ8nishMVf2z69WZDk9V2XGgguydRWR7Qz4nv4Qa7yib5JhODE+PZ/Jp3RjR02nN2wyRxrSML109t6jqU4eeqOpBEbkFOGHwi8idwC04XUTPqOoTIpIEvApkArnAVap68BRqN+3QgfIaVucVkb2jiNXeFv3BCucPyMjwEE7rEc8NZ/ViRM8ERqQnkJ4YZf3yxrQyX4I/VETk0E1YRCQU6HSijURkGE7ojwFqgPe9t26chXM7x9+JyH3AfcDPTvUHMG1XjaeetbuKD7fks3cWHe6XF4EBaXFcOKQrI3omMLJnAgO6xNqYeWP8wJfgfx94VUSe9j6/FXjPh+0GA18eutBLRBYD04CpwETvOs8Dn2LB3yGoKrmFFSzZVMCSTQUs21ZIRU0dAN3iIxnZM4FrzsxgZM8EhvWIJ9YmJTMmIHz5n/cznFb6bd7na3BG9pxIDvCwiCTjDP+cjDP9QxdV3e1dZw/QpbGNRWSW97hkZGT4cDgTCCVVtSzdsp8lm/ezZFMBeQedkb69kqOZdnoPzu6XwqiMRLp0tgujjGkrfBnVUy8iXwJ9gauAFOANH7bbICKP4FzlWw5kA3XHrKMi0uh9fFV1LjAXICsry+7120bU1Str8opYsmk/n20u4OudRdTVK7ERYZzVN5lbz+3LhP4p9EqOCXSpxpgmNBn8IjIAmOn92o9zQhZVneTrzlV1HjDPu7/f4kzpvFdEuqnqbhHpBuw79fKNP+QXVbJkUwGfbd7Pv7fsp7iyFhE4rUc8t5/blwkDUhmVkUC49c8b0y401+LfCHwGXKaqWwBE5O6T2bmIpKnqPhHJwOnfHwv0Bm4Afuf9/s9TKdy4a+OeEt7OzufD9XvZss+ZsaNL5wguHNKFcwakcna/FJJiTniO3xjTBjUX/NOAGcAnIvI+MB9nWObJeMPbx18L3KGqRSLyO+A1EbkZ2I7TfWTagB2FFby9ehdvr85n094yQkOEsX2SmHFGT87pn8qALrE2tNKYDqDJ4FfVBcACEYnBGYlzF5AmInOAt1T1wxPtXFXPaWRZIXD+qZdsWtO+kioWrtnN26vzWb2zCICsXok8NHUok0/rZhdLGdMB+XJytxx4GXhZRBKBK3FG+pww+E3bVFRRw3s5e3g7O58vvi1EFYZ068x9lwzisuHdSE+0yVeN6chOaiC19wrbw6NtTPtRXu3h4w17eTs7nyWbC6itU3qnxPCf5/Vnyoju9EuLDXSJxhg/sStoOrDauno+/aaAf2bvYtGGfVTW1tEtPpIbx2UyZUQPhvXobH32xgQhC/4OqL5eWbgmn99/tInthRUkRocz7fQeTBnRnTMykwgJsbA3JphZ8Hcgqson3+zjsQ82sWF3CYO6xvH09aM5b1CajbE3xhxmwd9BfPXtAR77YCPLcw/SKzmaP84YyXeHd7fWvTHmOBb87dy6/GIe/+AbPvmmgLS4CB6+YhhXZfW0Fr4xpkkW/O1U7v5y/uejTSxcnU98VDj3XTKIG87KJKqT3W7QGNM8C/52Zm9JFX9ctJnXlu8kPDSEOyb1ZdaEvsRH2a0HjTG+seBvJ4oqapjz6VaeW5pLvSrXnpnBHef1Iy3Opjs2xpwcC/42rrzaw7Off8vTS7ZRVu3hipE9uPs7A+iZZFfXGmNOjQV/G/bBuj388q0c9pdVc8HgLtx70QAGde0c6LKMMe2cBX8b9dKX27l/QQ6n9Yjn6etHM7pXYqBLMsZ0EBb8bYyq8uS/tvA/H23ivEFpPHXN6TZSxxjTqiz425D6euWhd9bz3NJcpo3qwSPTh9t4fGNMq7PgbyNqPPX85PXVLFydzw/P7s0vJg+2q26NMa6w4G8Dyqs93PbiSj7bvJ/7LhnErRP62KyZxhjXWPAH2IHyGn7w3HLW5hXx6PeGc9UZPQNdkjGmg3O1A1lE7haRdSKSIyKviEikiJwnIqu8y54XkaD95bOrqJLpf1nKxt0l/OW60Rb6xhi/cC34RaQH8GMgS1WHAaHANcDzwAzvsu3ADW7V0JZt3lvK9DlLKSit5u83jeHCoV0DXZIxJki4PWQkDIjytuqjgXKgRlU3eV//CPieyzW0Oat2HOTKp5fhqVdenXUWZ/ZJDnRJxpgg4lrwq+ou4HFgB7AbKAZeA8JEJMu72nSg0f4NEZklIitEZEVBQYFbZfrdJ9/s49pnviQ+Kpw3bhvHkO52Ja4xxr/c7OpJBKYCvYHuQAxwLTAD+IOIfAWUAnWNba+qc1U1S1WzUlNT3SrTrxZ8vYtbnl9B75QY/nHbODKSbb4dY4z/uXli9QLgW1UtABCRN4FxqvoicI532YXAABdraDP+9u9veeid9Yztk8Tc72fROdKmUTbGBIabffw7gLEiEi3OoPTzgQ0ikgYgIhHAz4C/uFhDwKkqj32wkYfeWc/FQ7vy3A/GWOgbYwLKtRa/qn4pIv8AVgEe4GtgLvAbEbkM55fOHFX9l1s1BJqnrp77F+Qwf/lOZo7J4DeXDyPUrsY1xgSYqGqgazihrKwsXbFiRaDLOClVtXXcOf9rPli3l/88rx/3fGeAXY1rjPErEVmpqlnHLg/ai6fcVFpVy6y/r2TZtkIeuGwIN53dO9AlGWPMYRb8rWx/WTU3PvsVG3eX8sTVI7l8VI9Al2SMMUex4G9FeQcruH7eV+wuruSZ72cxaVBaoEsyxpjjWPC3kk17S7l+3pdU1tTx4s1nkpWZFOiSjDGmURb8rWDl9oPc9NxyIsJCeO22s+y+uMaYNs2Cv4UWbyrgthdWktY5ghdvPpOeSXY1rjGmbbPgb4G3V+fzk9ey6Z8Wx/M3jSE1LiLQJRljzAlZ8J+ivy/L5cG313FGZhJ/vcGmYDDGtB8W/CdJVXni4838cdFmLhjchSevGUVkeGigyzLGGJ9Z8J+E+npl9sJ1/H3ZdqaPTud3004jLNTtWxoYY0zrsuD3UY2nnntfX83bq/OZNaEPP79kkE3BYIxplyz4fVBR4+G2F1exZFMB910yiNvO7Rvokowx5pRZ8J9AUUUNP3huOat3FvHo94bbDdGNMe2eBX8z9pZUcf28L8ktrGDOdaO5yG6IbozpACz4m3H/ghzyDlby3A/OYFzflECXY4wxrcKGpDRh454SPlq/l1kT+ljoG2M6FAv+Jsz5dCsxnUK5cVxmoEsxxphWZcHfiNz95Sxcnc91Y3uREN0p0OUYY0yrcjX4ReRuEVknIjki8oqIRIrI+SKySkSyReTfItLPzRpOxdNLthIWGsLNducsY0wH5Frwi0gP4MdAlqoOA0KBGcAc4FpVHQm8DNzvVg2nYk9xFf9YmcfVWT1J6xwZ6HKMMabVuT2qJwyIEpFaIBrIBxQ4NGF9vHdZmzF3yTbqFWZN6BPoUozpkGpra8nLy6OqqirQpXQYkZGRpKenEx7u22SRrgW/qu4SkceBHUAl8KGqfigiPwTeFZFKoAQY29j2IjILmAWQkZHhVplHKSyr5pWvdnD5yB42r74xLsnLyyMuLo7MzEyb9qQVqCqFhYXk5eXRu7dv3dNudvUkAlOB3kB3IEZErhha2NoAABAKSURBVAPuBiarajrwLPD7xrZX1bmqmqWqWampqW6VeZRnP8+lylPH7ROttW+MW6qqqkhOTrbQbyUiQnJy8kn9BeXmyd0LgG9VtUBVa4E3gfHACFX90rvOq8A4F2vwWUlVLc8vy+XioV3plxYX6HKM6dAs9FvXyb6fbgb/DmCsiESLU9X5wHogXkQGeNf5DrDBxRp89uIX2ymt8nDHpDY3yMgYY1qVa8HvbdX/A1gFrPUeay5wC/CGiKwGrgf+y60afFVZU8e8z77l3AGpDOsRH+hyjDEuKiwsZOTIkYwcOZKuXbvSo0ePw89ramqa3XbFihX8+Mc/PuExxo1rEx0ZTXJ1VI+qPgg8eMzit7xfbcary3dQWF7Dj86z1r4xHV1ycjLZ2dkAzJ49m9jYWO69997Dr3s8HsLCGo/GrKwssrKyTniMpUuXtk6xLgn6SdpqPPU8vWQbYzKTOCMzKdDlGBNUfrVwHevzS1p1n0O6d+bB7w49qW1uvPFGIiMj+frrrxk/fjwzZszgzjvvpKqqiqioKJ599lkGDhzIp59+yuOPP84777zD7Nmz2bFjB9u2bWPHjh3cddddh/8aiI2NpaysjE8//ZTZs2eTkpJCTk4Oo0eP5sUXX0REePfdd7nnnnuIiYlh/PjxbNu2jXfeeadV34umBH3wL/h6F7uLq/h/004LdCnGmADKy8tj6dKlhIaGUlJSwmeffUZYWBgff/wxv/jFL3jjjTeO22bjxo188sknlJaWMnDgQG6//fbjxtJ//fXXrFu3ju7duzN+/Hg+//xzsrKyuPXWW1myZAm9e/dm5syZ/voxgSAP/rp6Zc7irQzr0ZlzB/hnyKgx5oiTbZm76corryQ0NBSA4uJibrjhBjZv3oyIUFtb2+g2l156KREREURERJCWlsbevXtJT08/ap0xY8YcXjZy5Ehyc3OJjY2lT58+h8fdz5w5k7lz57r40x0tqCdpe3ftbr7dX84dE/vZ8DJjglxMTMzhx//93//NpEmTyMnJYeHChU2OkY+IiDj8ODQ0FI/Hc0rr+FvQBr+q8tQnW+ibGmN31jLGHKW4uJgePXoA8Nxzz7X6/gcOHMi2bdvIzc0F4NVXX231YzQnaIP/Xxv3sXFPKf8xsR8hIdbaN8Yc8dOf/pSf//znjBo1ypUWelRUFH/+85+5+OKLGT16NHFxccTH+28ouaiq3w52qrKysnTFihWttj9VZdqcpRSUVvPJvRMJDw3a33/G+N2GDRsYPHhwoMsIuLKyMmJjY1FV7rjjDvr378/dd999yvtr7H0VkZWqetz406BMvC+2HeDrHUXcem5fC31jTEA888wzjBw5kqFDh1JcXMytt97qt2MH5aiepz7ZQkpsBFeOTj/xysYY44K77767RS38lgi65m72ziL+vWU/t5zTm8jw0ECXY4wxfhd0wf/UJ1uIjwrn2rG9Al2KMcYERFAF/zd7Svlo/V5uHJdJbERQ9nIZY0xwBf+cT7cQ3SmUH4zPDHQpxhgTMEET/NsLy3l7dT7Xje1FQnSnQJdjjAmQSZMm8cEHHxy17IknnuD2229vdP2JEydyaDj55MmTKSoqOm6d2bNn8/jjjzd73AULFrB+/frDzx944AE+/vjjky2/VQRN8P9l8TbCQkP44dm+3ZPSGNMxzZw5k/nz5x+1bP78+T5NlPbuu++SkJBwSsc9NvgfeughLrjgglPaV0sFRUf3nuIq3liZx1VnpJPWOTLQ5RhjDnnvPtiztnX32fU0uOR3Tb48ffp07r//fmpqaujUqRO5ubnk5+fzyiuvcM8991BZWcn06dP51a9+ddy2mZmZrFixgpSUFB5++GGef/550tLS6NmzJ6NHjwac8flz586lpqaGfv368cILL5Cdnc3bb7/N4sWL+c1vfsMbb7zBr3/9ay677DKmT5/OokWLuPfee/F4PJxxxhnMmTOHiIgIMjMzueGGG1i4cCG1tbW8/vrrDBo0qMVvUVC0+J/5bBt1qtw6oW+gSzHGBFhSUhJjxozhvffeA5zW/lVXXcXDDz/MihUrWLNmDYsXL2bNmjVN7mPlypXMnz+f7Oxs3n33XZYvX374tWnTprF8+XJWr17N4MGDmTdvHuPGjWPKlCk89thjZGdn07fvkSyqqqrixhtv5NVXX2Xt2rV4PB7mzJlz+PWUlBRWrVrF7bfffsLuJF91+Bb/gfIaXv5yB1NHdqdnUnSgyzHGNNRMy9xNh7p7pk6dyvz585k3bx6vvfYac+fOxePxsHv3btavX8/w4cMb3f6zzz7jiiuuIDrayZQpU6Ycfi0nJ4f777+foqIiysrKuOiii5qt5ZtvvqF3794MGODcivyGG27gqaee4q677gKcXyQAo0eP5s0332zxzw4ut/hF5G4RWSciOSLyiohEishnIpLt/coXkQVu1vDs599S5anjPyZaa98Y45g6dSqLFi1i1apVVFRUkJSUxOOPP86iRYtYs2YNl156aZNTMZ/IjTfeyJNPPsnatWt58MEHT3k/hxya1rk1p3R2LfhFpAfwYyBLVYcBocAMVT1HVUeq6khgGdA6v8IaUVpVy3NLc7loSFf6pcW5dRhjTDsTGxvLpEmTuOmmm5g5cyYlJSXExMQQHx/P3r17D3cDNWXChAksWLCAyspKSktLWbhw4eHXSktL6datG7W1tbz00kuHl8fFxVFaWnrcvgYOHEhubi5btmwB4IUXXuDcc89tpZ+0cW738YcBUSISBkQD+YdeEJHOwHmAay3+F77YTmmVhzsm2U3UjTFHmzlzJqtXr2bmzJmMGDGCUaNGMWjQIK655hrGjx/f7Lann346V199NSNGjOCSSy7hjDPOOPzar3/9a84880zGjx9/1InYGTNm8NhjjzFq1Ci2bt16eHlkZCTPPvssV155JaeddhohISHcdtttrf8DN+DqtMwicifwMFAJfKiq1zZ47fvAFFWd3sS2s4BZABkZGaO3b99+0sd/fcVOluce4NHpI06lfGOMC2xaZne0iWmZRSQRmAr0BroDMSJyXYNVZgKvNLW9qs5V1SxVzUpNPbX74V6Z1dNC3xhjjuFmV88FwLeqWqCqtTh9+eMARCQFGAP8n4vHN8YY0wg3g38HMFZEosW5k/n5wAbva9OBd1S1Zae7jTHtUnu48197crLvp2vBr6pfAv8AVgFrvcea6315Bs108xhjOq7IyEgKCwst/FuJqlJYWEhkpO+zErh6AZeqPgg82MjyiW4e1xjTdqWnp5OXl0dBQUGgS+kwIiMjSU/3/Y6CHf7KXWNM2xIeHk7v3jZZYiAFxVw9xhhjjrDgN8aYIGPBb4wxQcbVK3dbi4gUACd/6a5/pAD7A11EM6y+lrH6Wsbqa7mW1NhLVY+7ArZdBH9bJiIrGrskuq2w+lrG6msZq6/l3KjRunqMMSbIWPAbY0yQseBvubknXiWgrL6WsfpaxupruVav0fr4jTEmyFiL3xhjgowFvzHGBBkLfh+ISE8R+URE1ntvHn9nI+tMFJHiBjeSf8DPNeaKyFrvsVc08rqIyP+KyBYRWSMip/uxtoEN3pdsESkRkbuOWcev75+I/E1E9olIToNlSSLykYhs9n5PbGLbG7zrbBaRG/xY32MistH77/eWiCQ0sW2znwUX65stIrsa/BtObmLbi0XkG+9n8T4/1vdqg9pyRSS7iW398f41mil++wyqqn2d4AvoBpzufRwHbAKGHLPORJx7DASqxlwgpZnXJwPvAQKMBb4MUJ2hwB6cC0sC9v4BE4DTgZwGyx4F7vM+vg94pJHtkoBt3u+J3seJfqrvQiDM+/iRxurz5bPgYn2zgXt9+PffCvQBOgGrj/2/5FZ9x7z+P8ADAXz/Gs0Uf30GrcXvA1XdraqrvI9LcW4o0yOwVZ20qcDf1fEFkCAi3QJQx/nAVlUN6JXYqroEOHDM4qnA897HzwOXN7LpRcBHqnpAVQ8CHwEX+6M+Vf1QVT3ep18Avs/D28qaeP98MQbYoqrbVLUGmI/zvreq5urz3hjqKgJ4T5BmMsUvn0EL/pMkIpnAKODLRl4+S0RWi8h7IjLUr4WBAh+KyErvjeqP1QPY2eB5HoH55dXcTXgC+f4BdFHV3d7He4AujazTVt7Hm3D+gmvMiT4LbvqRtyvqb010U7SF9+8cYK+qbm7idb++f8dkil8+gxb8J0FEYoE3gLtUteSYl1fhdF+MAP4ELPBzeWer6unAJcAdIjLBz8c/IRHpBEwBXm/k5UC/f0dR52/qNjnWWUR+CXiAl5pYJVCfhTlAX2AksBunO6UtmknzrX2/vX/NZYqbn0ELfh+JSDjOP9BLqvrmsa+raomqlnkfvwuEi3NTeb9Q1V3e7/uAt3D+pG5oF9CzwfN07zJ/ugRYpap7j30h0O+f195D3V/e7/saWSeg76OI3AhcBlzrDYbj+PBZcIWq7lXVOlWtB55p4riBfv/CgGnAq02t46/3r4lM8ctn0ILfB94+wXnABlX9fRPrdPWuh4iMwXlvC/1UX4yIxB16jHMSMOeY1d4Gvu8d3TMWKG7wJ6W/NNnSCuT718DbwKEREjcA/2xknQ+AC0Uk0duVcaF3metE5GLgp8AUVa1oYh1fPgtu1dfwnNEVTRx3OdBfRHp7/wKcgfO++8sFwEZVzWvsRX+9f81kin8+g26eue4oX8DZOH9yrQGyvV+TgduA27zr/AhYhzNK4QtgnB/r6+M97mpvDb/0Lm9YnwBP4YyoWAtk+fk9jMEJ8vgGywL2/uH8AtoN1OL0kd4MJAOLgM3Ax0CSd90s4K8Ntr0J2OL9+oEf69uC07d76DP4F++63YF3m/ss+Km+F7yfrTU4Adbt2Pq8zyfjjGLZ6s/6vMufO/SZa7BuIN6/pjLFL59Bm7LBGGOCjHX1GGNMkLHgN8aYIGPBb4wxQcaC3xhjgowFvzHGBBkLfmMAEamTo2cQbbVZI0Uks+EskcYEWligCzCmjahU1ZGBLsIYf7AWvzHN8M7N/qh3fvavRKSfd3mmiPzLOyHZIhHJ8C7vIs5c+au9X+O8uwoVkWe8c69/KCJRAfuhTNCz4DfGEXVMV8/VDV4rVtXTgCeBJ7zL/gQ8r6rDcSZL+1/v8v8FFqsz2dzpOFd/AvQHnlLVoUAR8D2Xfx5jmmRX7hoDiEiZqsY2sjwXOE9Vt3kn1dqjqskish9nSoJa7/LdqpoiIgVAuqpWN9hHJs786f29z38GhKvqb9z/yYw5nrX4jTkxbeLxyahu8LgOO79mAsiC35gTu7rB92Xex0txZpYEuBb4zPt4EXA7gIiEiki8v4o0xlfW6jDGESVH33z7fVU9NKQzUUTW4LTaZ3qX/SfwrIj8F1AA/MC7/E5grojcjNOyvx1nlkhj2gzr4zemGd4+/ixV3R/oWoxpLdbVY4wxQcZa/MYYE2SsxW+MMUHGgt8YY4KMBb8xxgQZC35jjAkyFvzGGBNk/j+NhLEL4nKEAQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zeTFBOtlgzc5",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "4516d28b-5026-48b1-e041-8cd0802b28a0"
      },
      "source": [
        "with torch.set_grad_enabled(False):\n",
        "    test_acc = compute_acc(model=model,\n",
        "                           data_loader=test_loader,\n",
        "                           device=DEVICE)\n",
        "    \n",
        "    valid_acc = compute_acc(model=model,\n",
        "                            data_loader=valid_loader,\n",
        "                            device=DEVICE)\n",
        "    \n",
        "\n",
        "print(f'Validation ACC: {valid_acc:.2f}%')\n",
        "print(f'Test ACC: {test_acc:.2f}%')"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Validation ACC: 94.30%\n",
            "Test ACC: 91.63%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dghQX5GLgzc8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        },
        "outputId": "e7db0c66-2db0-497c-b792-593ad558e4ae"
      },
      "source": [
        "%watermark -iv"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "numpy 1.18.5\n",
            "torch 1.5.1+cu101\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}